add thread

xuming06 · xuming06 · commit b232d087b028 · 2018-07-03T22:38:24.000+08:00
diff --git a/07keras/11lstm_text_generation.py b/07keras/11lstm_text_generation.py
@@ -62,8 +62,10 @@ def get_corpus(data_path):
 model.add(Activation('softmax'))
 
 model.compile(optimizer=RMSprop(lr=0.01), loss='categorical_crossentropy')
+model.summary()
 
-
+print("*"*40)
+print(model.summary())
 def sample(preds, temperature=1.0):
     preds = np.asarray(preds).astype('float64')
     preds = np.log(preds) / temperature
diff --git a/12gensim/04.doc2vec.py b/12gensim/04.doc2vec.py
@@ -22,9 +22,9 @@
 from sklearn.linear_model import LogisticRegression
 
 # 获取训练与测试数据及其类别标注
-neg_file = 'neg.txt'
-pos_file = 'douban_imdb_data/aclImdb/train/pos'
-unsup_file = 'douban_imdb_data/aclImdb/train/unsup'
+neg_file = '../data/douban_imdb_data/neg.txt'
+pos_file = '../data/douban_imdb_data/pos.txt'
+unsup_file = '../data/douban_imdb_data/unsup.txt'
 sentences = gensim.models.doc2vec.TaggedLineDocument(neg_file)
 model = gensim.models.doc2vec.Doc2Vec(sentences)
 model.save('neg.d2v.model')
@@ -39,7 +39,6 @@
 print(model.doesnt_match("I'm sure I missed some plot points".split()))
 
 
-print(model.most_similar(positive=['but', 'what'], negative=['fact']))
 print(model.most_similar(positive=['blue', 'shirt'], negative=['blue']))
 
 
diff --git a/12gensim/04.doc2vec_demo.py b/12gensim/04.doc2vec_demo.py
@@ -20,13 +20,10 @@ def get_data(pos_file, neg_file, unsup_file):
     :return: 
     """
 
-    def get_folder_txt(folder_path):
+    def get_folder_txt(path):
         result = []
-        for parent, dirnames, filenames in os.walk(folder_path):
-            for filename in filenames:
-                path = os.path.join(folder_path, filename)
-                with open(path, 'r', encoding='utf-8') as f:
-                    result.append(f.read())
+        with open(path, 'r', encoding='utf-8') as f:
+            result.append(f.read())
         return result
 
     pos_reviews = get_folder_txt(pos_file)
@@ -185,9 +182,9 @@ def ROC_curve(lr, y_test):
     # 设置向量维度和训练次数
     size, epoch_num = 400, 10
     # 获取训练与测试数据及其类别标注
-    neg_file = 'douban_imdb_data/aclImdb/train/neg'
-    pos_file = 'douban_imdb_data/aclImdb/train/pos'
-    unsup_file = 'douban_imdb_data/aclImdb/train/unsup'
+    neg_file = '../data/douban_imdb_data/neg.txt'
+    pos_file = '../data/douban_imdb_data/pos.txt'
+    unsup_file = '../data/douban_imdb_data/unsup.txt'
     x_train, x_test, unsup_reviews, y_train, y_test = get_data(neg_file, pos_file, unsup_file)
     # 对数据进行训练，获得模型
     model_dm, model_dbow = train(x_train, x_test, unsup_reviews, size, epoch_num)
diff --git a/12gensim/doc2v.py b/12gensim/doc2v.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Author: XuMing <xuming624@qq.com>
+# Brief: 
+# coding:utf-8
+
+import sys
+import gensim
+import sklearn
+import numpy as np
+
+from gensim.models.doc2vec import Doc2Vec, LabeledSentence
+
+TaggededDocument = gensim.models.doc2vec.TaggedDocument
+
+
+def get_datasest():
+    with open("../data/ngram_wordseg/gold.txt", 'r', encoding='utf8') as cf:
+        docs = cf.readlines()
+        print(len(docs))
+
+    x_train = []
+    # y = np.concatenate(np.ones(len(docs)))
+    for i, text in enumerate(docs):
+        word_list = text.split()
+        l = len(word_list)
+        word_list[l - 1] = word_list[l - 1].strip()
+        document = TaggededDocument(word_list, tags=[i])
+        x_train.append(document)
+
+    return x_train
+
+
+def getVecs(model, corpus, size):
+    vecs = [np.array(model.docvecs[z.tags[0]].reshape(1, size)) for z in corpus]
+    return np.concatenate(vecs)
+
+
+def train(x_train, size=200, epoch_num=1):
+    model_dm = Doc2Vec(x_train, min_count=1, window=3, size=size, sample=1e-3, negative=5, workers=4)
+    model_dm.train(x_train, total_examples=model_dm.corpus_count, epochs=70)
+    model_dm.save('model_dm')
+
+    return model_dm
+
+
+def test():
+    model_dm = Doc2Vec.load("model_dm")
+    test_text = ['《', '舞林', '争霸' '》', '十强' '出炉', '复活', '舞者', '澳门', '踢馆']
+    inferred_vector_dm = model_dm.infer_vector(test_text)
+    print(inferred_vector_dm)
+    sims = model_dm.docvecs.most_similar([inferred_vector_dm], topn=10)
+
+    return sims
+
+
+if __name__ == '__main__':
+    x_train = get_datasest()
+    model_dm = train(x_train)
+
+    sims = test()
+    for count, sim in sims:
+        sentence = x_train[count]
+        words = ''
+        for word in sentence[0]:
+            words = words + word + ' '
+        print(words, sim, len(sentence[0]))
diff --git a/22data-mining/classifier_model_stack.py b/22data-mining/classifier_model_stack.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Author: XuMing <xuming624@qq.com>
+# Brief: 
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import itertools
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+from sklearn.ensemble import RandomForestClassifier
+from mlxtend.classifier import EnsembleVoteClassifier
+from mlxtend.data import iris_data
+from mlxtend.plotting import plot_decision_regions
+
+# Initializing Classifiers
+clf1 = LogisticRegression(random_state=0)
+clf2 = RandomForestClassifier(random_state=0)
+clf3 = SVC(random_state=0, probability=True)
+eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[2, 1, 1], voting='soft')
+
+# Loading some example data
+X, y = iris_data()
+X = X[:,[0, 2]]
+
+# Plotting Decision Regions
+gs = gridspec.GridSpec(2, 2)
+fig = plt.figure(figsize=(10, 8))
+
+for clf, lab, grd in zip([clf1, clf2, clf3, eclf],
+                         ['Logistic Regression', 'Random Forest', 'RBF kernel SVM', 'Ensemble'],
+                         itertools.product([0, 1], repeat=2)):
+    clf.fit(X, y)
+    ax = plt.subplot(gs[grd[0], grd[1]])
+    fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2)
+    plt.title(lab)
+    plt.savefig('1.png')
+plt.show()
diff --git a/22data-mining/frequent_patterns/dataSet/dblpDataAll.txt b/22data-mining/frequent_patterns/dataSet/dblpDataAll.txt
@@ -2,7 +2,7 @@ Michael Ley
 E. F. Codd
 E. F. Codd,C. J. Date
 Patrick A. V. Hall
-E. F. Codd
+E. F. Codd Codd
 Markus Tresch
 E. F. Codd
 E. F. Codd
diff --git a/22data-mining/frequent_patterns/freq_utils.py b/22data-mining/frequent_patterns/freq_utils.py
@@ -58,55 +58,35 @@ def loadUnixData(fileRead, fileWrite):
     return dataSet
 
 
-def getAuthorsData(fileRead, fileWrite):
+def load_title_data(file_path, flag, row_num=1):
     '''
-    加载原始作者数据预处理
-    :param fileName:
-    :return:
-    '''
-    f = open(fileRead, 'r')
-    fwrite = open(fileWrite, "w")
-    dataSet = []
-    i = 0
-    for line in f.readlines():
-        if line == "\n":
-            continue
-        line = line[:len(line) - 2]
-        line_arr = line.strip().split(',')
-        dataSet.append(line_arr)
-        fwrite.write(line + "\n")
-    return dataSet
-
-
-def getUnixData(fileRead, fileWrite):
-    '''
-    加载数据Unix用户命令数据
-    :param fileName:
+    加载title的数据
+    :param file_path:
     :return:
     '''
-    f = open(fileRead, 'r')
-    fwrite = open(fileWrite, "w")
+    dataSetDict = {}
     dataSet = []
-    temp = ''
-    for line in f.readlines():
-        if line == "\n":
-            continue
-        line = line.split("\n")[0]
-        print(line)
-        if line == "**SOF**":
-            temp = ''
-        elif line == "**EOF**":
-            if temp == "":
-                continue
-            fwrite.write(temp + "\n")
-        else:
-            if temp == "":
-                temp = line
-            else:
-                temp = temp + ',' + line
+    count = 0
+    print(file_path)
+    with open(file_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            if count > row_num:
+                break
+            line = list(line.strip())
+            dataSet.append(line)
+            dataLine = [word for word in line]
+            dataSetDict[frozenset(dataLine)] = dataSetDict.get(frozenset(dataLine), 0) + 1
+            count += 1
+    return dataSetDict, dataSet
 
 
 def printDataSet(dataSet):
     for i in range(len(dataSet)):
         for j in range(len(dataSet[i])):
             print(dataSet[i][j])
+
+
+def save_freqItems(freqItems_fp, save_path):
+    with open(save_path, 'w', encoding='utf-8') as f:
+        for i in freqItems_fp:
+            f.write(' '.join(i[0]) + '\t' + str(i[1]) + '\n')
diff --git a/22data-mining/frequent_patterns/main.py b/22data-mining/frequent_patterns/main.py
@@ -3,10 +3,11 @@
 # Brief:
 import time
 from eclat import eclat_zc
-from freq_utils import loadDblpData, loadData, loadUnixData
+from freq_utils import loadDblpData, load_title_data, printDataSet, save_freqItems
 from apriori import apriori_zc
 from fp_growth import fp_growth
 
+
 def test_fp_growth(minSup, dataSetDict, dataSet):
     freqItems = fp_growth(dataSetDict, minSup)
     freqItems = sorted(freqItems.items(), key=lambda item: item[1])
@@ -32,6 +33,7 @@ def print_freqItems(logo, freqItems):
     print(len(freqItems))
     print("-------------------", logo, " end ---------------")
 
+
 def do_experiment_data_size():
     data_name = 'unixData8_pro.txt'
     x_name = "Data_Size"
@@ -151,7 +153,7 @@ def do_test():
 def do_dblp_data():
     data_name = 'dblpDataAll.txt'
     x_name = "Min_Support"
-    data_num = 2715700
+    data_num = 980
     minSup = 100
     dataSetDict, dataSet = loadDblpData(("dataSet/" + data_name), ',', data_num)
 
@@ -165,9 +167,27 @@ def do_dblp_data():
         print(item)
 
 
-if __name__ == '__main__':
-    x_value, y_value = do_experiment_min_support()
-    x_value, y_value = do_experiment_data_size()
-    do_test()
+def do_title_data():
+    data_name = 'title.txt'
+    x_name = "Min_Support"
+    data_num = 22846
+    minSup = data_num / 100
+    dataSetDict, dataSet = load_title_data(("dataSet/" + data_name), ',', data_num)
+    printDataSet(dataSet[:10])
+    time_fp = 0
+    ticks0 = time.time()
+    freqItems_fp = test_eclat(minSup, dataSetDict, dataSet)
+    time_fp += time.time() - ticks0
+    print(time_fp)
+
+    print(freqItems_fp[:10])
+    save_freqItems(freqItems_fp, "dataSet/title_out.txt")
 
+
+if __name__ == '__main__':
+    # x_value, y_value = do_experiment_min_support()
+    # x_value, y_value = do_experiment_data_size()
+    # do_test()
+    #
     do_dblp_data()
+    # do_title_data()
diff --git a/22data-mining/prob_stack.py b/22data-mining/prob_stack.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Author: XuMing <xuming624@qq.com>
+# Brief:
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import itertools
+from sklearn.linear_model import LogisticRegression
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import cross_val_score
+from mlxtend.classifier import EnsembleVoteClassifier, StackingClassifier
+from mlxtend.data import iris_data
+from mlxtend.plotting import plot_decision_regions
+
+# Loading some example data
+X, y = iris_data()
+X = X[:, [0, 2]]
+
+clf1 = KNeighborsClassifier(n_neighbors=1)
+clf2 = RandomForestClassifier(random_state=1)
+clf3 = GaussianNB()
+lr = LogisticRegression()
+sclf = StackingClassifier(classifiers=[clf1, clf2, clf3],
+                          use_probas=True,
+                          average_probas=False,
+                          meta_classifier=lr)
+
+print('3-fold cross validation:\n')
+
+# Plotting Decision Regions
+gs = gridspec.GridSpec(2, 2)
+fig = plt.figure(figsize=(10, 8))
+
+for clf, lab, grd in zip([clf1, clf2, clf3, sclf],
+                           ['KNN',
+                            'Random Forest',
+                            'Naive Bayes',
+                            'StackingClassifier'],
+                           itertools.product([0, 1], repeat=2)):
+    clf.fit(X, y)
+    ax = plt.subplot(gs[grd[0], grd[1]])
+    fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2)
+    plt.title(lab)
+
+    scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
+    print("Accuracy: %0.2f (+/- %0.2f) [%s]"
+          % (scores.mean(), scores.std(), lab))
+    plt.savefig('2.png')
+plt.show()
+
diff --git a/22data-mining/words_pattern/search.py b/22data-mining/words_pattern/search.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Author: XuMing <xuming624@qq.com>
+# Brief: get special words by sentence
+
+sentences = [
+    "{甲状腺节结治疗哪家医院好}?郑州市第二中医院",
+    "{检查前列腺囊仲对身体有害吗}前列腺囊仲的病因之一:久站久坐,性生活过频,性生活被迫中断.过多的手淫等.临床表现有排尿疫常等!",
+    "{龟头长小红点},在阴茎的龟头上有红斑点,红肿等切不可怠慢,可能是泌尿发炎症或性疾病所致.如龟头疾病,泡诊等泌尿疾病",
+    "现在处女模修复手术多少钱}<<沧州华美妇产医院>>韩式处女膜修复术,落红高",
+    "乳房乳腺节结手术吗}?南京京科医院,知名乳腺医师,美国先进检查设备,精准确诊乳腺节结",
+    "青岛市里医院{关键词}{能洗眼线吗}?选华韩整形洗眼线",
+    "男性朋友需警惕,冠状沟疾病是由于真菌感蒅,不洁性行为引起!",
+    "乐清哪个看妇科焱症的医院好}同济妇科医院,老百姓的妇科医院",
+    "信阳妇科医院} 专业开展无疼流产手术,技术治疗妇科炎症,玑瘤肿囊,女性不孕",
+    ",常见的臯丸仲胀,臯丸疼痛不适等.都可能是臯丸焱所引起的",
+    "中医治疗鱼鳞痔的方法},西安北方中医皮肤病医院,西安鱼鳞痔跈疗基地",
+]
+for i in sentences:
+    print(i)
diff --git a/data/douban_imdb_data/pos.txt b/data/douban_imdb_data/pos.txt
diff --git a/data/douban_imdb_data/unsup.txt b/data/douban_imdb_data/unsup.txt
