# -*- coding: utf-8 -*-
"""HW11.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1viERE20FyvePWsxg50qDw-zJIjo2S7Pk
"""

! wget https://www.aozora.gr.jp/cards/000148/files/752_14964.html -O sample.html

from bs4 import BeautifulSoup
soup = BeautifulSoup(open("sample.html",encoding="shift_jis"))
main_text=soup.find("div","main_text").text#<div>を取り出す


m1 = MeCab.Tagger("")
result = m1.parse(main_text)
Frequency = {}
Lines = result.splitlines()
for l in Lines[:-1]:
 word, pos = l.split('\t')
 posDetail = pos.split(',')
 if '名詞' in posDetail[0]:
   Frequency[word] = Frequency.get(word, 0) + 1

D = sorted(Frequency.items(),key=lambda x:x[1],reverse=True)
Frequency.clear()
Frequency.update(D)

from google.colab import files
with open('sample-output.txt','w') as f:
 for k,v in Frequency.items():
   f.write(str(k)+"  "+str(v)+'\n')
files.download('sample-output.txt')
5a6e0563fb2b33efefa6ea13e6d673cdc77d6980
# -*- coding: utf-8 -*-
"""HW11.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1viERE20FyvePWsxg50qDw-zJIjo2S7Pk
"""

! wget https://www.aozora.gr.jp/cards/000148/files/752_14964.html -O sample.html

from bs4 import BeautifulSoup
soup = BeautifulSoup(open("sample.html",encoding="shift_jis"))
main_text=soup.find("div","main_text").text#<div>を取り出す


m1 = MeCab.Tagger("")
result = m1.parse(main_text)
Frequency = {}
Lines = result.splitlines()
for l in Lines[:-1]:
 word, pos = l.split('\t')
 posDetail = pos.split(',')
 if '名詞' in posDetail[0]:
   Frequency[word] = Frequency.get(word, 0) + 1

D = sorted(Frequency.items(),key=lambda x:x[1],reverse=True)
Frequency.clear()
Frequency.update(D)

from google.colab import files
with open('sample-output.txt','w') as f:
 for k,v in Frequency.items():
   f.write(str(k)+"  "+str(v)+'\n')
files.download('sample-output.txt')