sk_主题模型

sk_主题模型

# from sklearn.datasets import fetch_20newsgroups
# dataset = fetch_20newsgroups(shuffle=True, random_state=1,
#                              remove=('headers', 'footers', 'quotes'))
# n_samples = 2000
# data_samples = dataset.data[:n_samples]
# print(data_samples[0:20])
import re
from collections import Counter
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
import jieba

filename = r"小程序query.xlsx"
save = "save.xlsx"
most_com = 60
# 读出数据
df = pd.read_excel(filename,index=None)
df_query = list(df["text"])
print(df_query[:10])
# 去除标签
def del_query_label(df_query):
    col = []
    for i in df_query:
        col.append(re.sub("\[.*\]","",i))
    return col

query_list = del_query_label(df_query)

seg_list = jieba.cut("今天收盘价是多少", cut_all=False)
print(seg_list)
res = Counter(seg_list)
print(res)

# jieba分词
def jieba_bat(query_list):
    col = []
    for i in query_list:
        col.extend(jieba.cut(i,cut_all=False))
    return col

word_list = jieba_bat(query_list)
print(word_list[:20])
# 统计
recmd = Counter(word_list)

# recmd_val = recmd.most_common(most_com)
recmd_val = recmd.most_common()
print(recmd_val)

#todo [('的', 2294), ('我', 2173), ('一下', 1297), ('帮', 1107), ('查', 1073)]写入excel

n = 0
df_d = {}
for i in zip(*recmd_val):
    df_d[n] = i
    n += 1
#todo 统计高频与长尾 一倍方差
# todo 累加求和   累积分布函数
total = sum(df_d[1])


def get_cdf(dict_df):
    total = sum(dict_df)
    col = []
    v = 0
    for i in dict_df:
        v+=i
        col.append(v/total)
    return col
    pass


df_d[2] = get_cdf(df_d[1])

df_d = pd.DataFrame(df_d)
df_d.to_excel(save,index=None)


# n_features = 20
# tf_vectorizer = CountVectorizer(max_df=5, min_df=5,
#                                 max_features=n_features,)
# tf = tf_vectorizer.fit_transform(seg_list)
# print(len(tf_vectorizer.vocabulary_))

　　统计CDF，累积分布函数（Cumulative Distribution Function,CDF）