# import package
import numpy as np
import pandas as pd
import collections, time, os, re
from itertools import chain
from collections import Counter
from nltk import collocations, everygrams, word_tokenize
from tqdm import tnrange, tqdm_notebook

# mecab 형태소분석 명사만
from konlpy.tag import Mecab
mecab = Mecab().nouns

# Twitter 형태소분석 명사만
from konlpy.utils import pprint
from konlpy.tag import Twitter
twitter = Twitter().nouns

# 시각화 라이브러리 한글폰트 설정
from matplotlib import font_manager, rc, pyplot as plt
import seaborn as sns
%matplotlib inline
font_name = font_manager.FontProperties(fname="/usr/share/fonts/truetype/nanum/NanumBarunGothicBold.ttf").get_name()
rc('font', family=font_name)
plt.rcParams["font.family"]

# pandas로 엑셀 파일 읽기
data = pd.read_excel('Keyword 빈도(Total Data 기준).xlsx', sheetname='Sheet3', header=None)
print("읽은 데이터 행 수 : ", len(data))
data["content"] = data[0].astype(str)
data["content"] = data["content"].str.replace("^"," ")

# 문의글 칼럼의 특수문자를 없애주고 preprocess_question 칼럼에 추가
data['preprocess_question']=[ re.sub('[\{\}\[\]\/?.,;:|\)*~`!^\-_+<>@\#$%&…▶◆\\\=ⓒ\(\'\"]','',i).strip() for i in data["content"] ]

# 2 어절을 추출해서 카운트 하는 방식 (3어절 이상도 가능)
print("ngram_start : ", ngram_start)
print("ngram_end : ", ngram_end)
ngram_data_question = data['preprocess_question'].apply(lambda x: [' '.join(ng) for ng in everygrams(word_tokenize(x), ngram_start, ngram_end)])

# ngram_question 칼럼에 추가
data['ngram_question'] = ngram_data_question

# 형태소 분석 칼럼 추가
data['pos'] = data['preprocess_question'].apply(pos_tool)
def joinlist(list):
    return " ".join(list)

# 형태소 분석 결과 한문장으로된 칼럼 추가
data['string_pos'] = data['pos'].apply(joinlist)


# 형태소 분석 명사만 추출 후 2 어절 카운트 (3어절 이상도 가능)
ngram_data_question_pos = data['string_pos'].apply(lambda x: [' '.join(ng) for ng in everygrams(word_tokenize(x), ngram_start, ngram_end)])

# ngram_question 칼럼에 추가
data['ngram_question_pos'] = ngram_data_question_pos

data = data.iloc[:,1:]
data.head(3)

heatmap_raw_data = data.ngram_question_pos.tolist()

# 카운터 사용
d = Counter(list(chain(*list(heatmap_raw_data))))
df_q = pd.DataFrame.from_dict(d, orient='index').reset_index()
df_q = df_q.sort_values(by=0, ascending=False)
print(ngram_start, "~", ngram_end, "어절 ngram 수", len(df_q))

print(heatmap_number, "x", heatmap_number, "히트맵 설정")

number = df_q['index'][:heatmap_number].tolist()
heatmap_word = []
heatmap_cnt = []

# 히트맵용 카운트
for i in tqdm_notebook(range(len(number))):
    heatmap_cnt = []
    
    for j in range(len(number)):
        if number[i] == number[j]:
            heatmap_cnt.append(0)
            break
        else:
            heatmap_cnt.append(len(data[(data['string_pos'].str.contains(number[i])) & (data['string_pos'].str.contains(number[j]))]))
    heatmap_word.append(heatmap_cnt)

# 히트맵용 카운트를 dataframe으로 변경
heatmap_data = pd.DataFrame(heatmap_word, index=number, columns=number)
heatmap_data = heatmap_data.fillna(0)
heatmap_data[heatmap_data.columns] = heatmap_data[heatmap_data.columns].astype(int)
heatmap_data.iloc[:heatmap_number,:heatmap_number]

fig, ax = plt.subplots(figsize=(heatmap_number/1.5, heatmap_number/2))
sns.heatmap(heatmap_data.iloc[:heatmap_number,:heatmap_number], cmap=colorset, annot=True, fmt="d", linewidths=.5)

# 특정단어를 포함한 ngram만 추출
df_q = df_q.rename(index=str, columns={"index": "ngram", 0: "count"})

# 전체 데이터 저장
df_q.to_csv("ngram.csv")

df_q.head(20)

df_q[df_q["ngram"].str.contains("파손")][:20].plot(x='ngram', y='count', kind='barh')

df_q[df_q["ngram"].str.contains("파손")].head()