[sklearn, SVM] 서포트 백터 머신을 이용해서 외국어 문장 판별하기

 

 

 

SVM 서포트 백터 머신을 이용해서 외국어 문장 판별하기¶

In [25]:

from sklearn import svm, metrics
import glob, os.path, re, json

In [15]:

def check_freq(fname):
    name = os.path.basename(fname)
    lang = re.match(r'^[a-z]{2,}',name).group()
    with open(fname, 'r', encoding='utf-8') as f:
        text = f.read()
    text = text.lower()
    cnt=[0 for n in range(0,26)]
    code_a = ord('a')  #97~
    code_z = ord('z')
    for ch in text:
        n = ord(ch) #숫자값
        if code_a <= n <= code_z:
            cnt[n - code_a] += 1
    total = sum(cnt)
    freq = list(map(lambda n: n/total, cnt))
    return (freq, lang)

In [16]:

def load_file(path):
    freqs = []
    labels = []
    file_list = glob.glob(path)
    for fname in file_list:
        r = check_freq(fname)
        freqs.append(r[0])
        labels.append(r[1])
    return {'freqs':freqs, 'labels':labels}

In [17]:

data = load_file('./lang/train/*.txt')
test = load_file('./lang/test/*.txt')

In [19]:

# 이후를 대비해서 JSON으로 결과 저장하기 - 이미지 출력용
with open("./lang/freq.json", "w", encoding="utf-8") as fp:
    json.dump([data, test], fp)

In [21]:

data['labels'][0]

Out[21]:

'en'

In [22]:

clf = svm.SVC()
clf.fit(data['freqs'], data['labels'])

Out[22]:

SVC()

In [23]:

predict = clf.predict(test['freqs'])

In [27]:

ac_score = metrics.accuracy_score(test['labels'], predict)
ac_score

Out[27]:

1.0

In [28]:

import matplotlib.pyplot as plt
import pandas as pd
import json

In [29]:

with open('lang/freq.json', 'r', encoding='utf-8') as f:
    freq = json.load(f)

In [30]:

freq

Out[30]:

[{'freqs': [[0.07595212187159957,
    0.012840043525571273,
    0.04570184983677911,
    0.04613710554951034,
    0.10533188248095757,
    0.015669205658324265,
    0.019151251360174103,
    0.043743199129488576,
    0.07399347116430903,
    0.0017410228509249185,
    0.00544069640914037,
    0.05375408052230685,
    0.026332970620239392,
    0.07747551686615888,
    0.08966267682263329,
    0.016539717083786723,
    0.0,
    0.07769314472252448,
    0.061371055495103376,
    0.08052230685527748,
    0.02589771490750816,
    0.009793253536452665,
    0.014145810663764961,
    0.0006528835690968443,
    0.02002176278563656,
    0.0004352557127312296],
   
  'labels': ['en', 'en', 'fr', 'fr', 'id', 'id', 'tl', 'tl']}]

In [33]:

# 언어마다 계산하기 --- (※2)
lang_dic = {}
for i, lbl in enumerate(freq[0]["labels"]):
    fq = freq[0]["freqs"][i]
    if not (lbl in lang_dic):
        lang_dic[lbl] = fq
        continue
    for idx, v in enumerate(fq):
        lang_dic[lbl][idx] = (lang_dic[lbl][idx] + v) / 2

In [34]:

# Pandas의 DataFrame에 데이터 넣기 --- (※3)
asclist = [[chr(n) for n in range(97,97+26)]]
df = pd.DataFrame(lang_dic, index=asclist)

In [35]:

df.head()

Out[35]:

	en	fr	id	tl
a	0.073919	0.076504	0.171599	0.201979
b	0.020681	0.012910	0.025640	0.022360
c	0.033506	0.036590	0.007429	0.015670
d	0.039112	0.050464	0.040608	0.027269
e	0.131043	0.148523	0.081552	0.055009

In [36]:

# 그래프 그리기 --- (※4)
df.plot(kind="bar", subplots=True, ylim=(0,0.15))
plt.savefig("lang-plot.png")

 

In [37]:

# 다른 그래프로 확인
df.plot(kind='line')

Out[37]:

<AxesSubplot:>