# -*- coding: utf-8 -*-
"""감성 분석

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1qQ50H_T1pDU6ZUANy_HFxxuddrLy_hvq
"""

import warnings
warnings.filterwarnings(action='ignore')

!pip install mxnet-cu101
!pip install gluonnlp padas tqdm
!pip install sentencepiece==0.1.85
!pip install transformers==2.1.1
!pip install torch==1.3.1

!pip install git+https://git@github.com/SKTBrain/KoBERT.git@master

!pip install gluonnlp

import torch
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import gluonnlp as nlp
import numpy as np
from tqdm import tqdm, tqdm_notebook

from kobert.utils import get_tokenizer
from kobert.pytorch_kobert import get_pytorch_kobert_model

from transformers import AdamW
from transformers.optimization import WarmupLinearSchedule

device = torch.device("cuda:0")

bertmodel, vocab = get_pytorch_kobert_model()

!wget https://www.dropbox.com/s/374ftkec978br3d/ratings_train.txt?dl=1
!wget https://www.dropbox.com/s/977gbwh542gdy94/ratings_test.txt?dl=1

dataset_train = nlp.data.TSVDataset("ratings_train.txt?dl=1", field_indices=[1,2], num_discard_samples=1)
dataset_test = nlp.data.TSVDataset("ratings_test.txt?dl=1", field_indices=[1,2], num_discard_samples=1)

tokenizer = get_tokenizer()
tok = nlp.data.BERTSPTokenizer(tokenizer,vocab,lower=False)

class BERTDataset(Dataset):
    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len, pad, pair):
        transform = nlp.data.BERTSentenceTransform(bert_tokenizer,max_seq_length=max_len,pad=pad,pair=pair)
        self.sentences = [transform([i[sent_idx]]) for i in dataset]
        self.labels = [np.int32(i[label_idx]) for i in dataset]

    def __getitem__(self,i):
        return (self.sentences[i] + (self.labels[i], ))

    def __len__(self):
        return (len(self.labels))

max_len = 64
batch_size = 64
warmup_ratio = 0.1
num_epochs = 5
max_grad_norm = 1
log_interval = 200
learning_rate = 5e-5

data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)
data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)

train_dataloader = torch.utils.data.DataLoader(data_train,batch_size=batch_size,num_workers=5)
test_dataloader = torch.utils.data.DataLoader(data_test,batch_size=batch_size,num_workers=5)

class BERTClassifier(nn.Module):
    def __init__(self,bert,hidden_size=768,num_classes=2,dr_rate=None,params=None):
        super(BERTClassifier,self).__init__()
        self.bert=bert
        self.dr_rate = dr_rate
        
        self.classifier = nn.Linear(hidden_size, num_classes)
        if dr_rate:
            self.dropout = nn.Dropout(p=dr_rate)

    def gen_attention_mask(self, token_ids, valid_length):
        attention_mask = torch.zeros_like(token_ids)
        for i,v in enumerate(valid_length):
            attention_mask[i][:v]=1
        return attention_mask.float()

    def forward(self, token_ids, valid_length, segment_ids):
        attention_mask = self.gen_attention_mask(token_ids,valid_length)

        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(),
                              attention_mask = attention_mask.float().to(token_ids.device))

        if self.dr_rate:
            out = self.dropout(pooler)
        
        return self.classifier(out)

model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)

# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]

optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
loss_fn = nn.CrossEntropyLoss()

t_total = len(train_dataloader) * num_epochs
warmup_step = int(t_total*warmup_ratio)

scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_step, t_total=t_total)

def calc_accuracy(X,Y):
    max_vals, max_indices = torch.max(X, 1)
    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]
    return train_acc

for e in range(num_epochs):
    train_acc=0.0
    test_acc=0.0
    model.train()

    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(train_dataloader)):
        optimizer.zero_grad()
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length = valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        loss = loss_fn(out, label)
        loss.backward()
        torch.nn.utils.clip_grad_norm(model.parameters(), max_grad_norm)
        optimizer.step()
        scheduler.step()

        if batch_id % log_interval == 0:
            print("epoch {} batch id {} loss {} train acc {}".format(e+1, batch_id+1, loss.data.cpu().numpy(), train_acc / (batch_id+1)))
    print("epoch {} train acc {}".format(e+1, train_acc / (batch_id+1)))
    model.eval()

    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(test_dataloader)):
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length= valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        test_acc += calc_accuracy(out, label)
    print("epoch {} test acc {}".format(e+1, test_acc / (batch_id+1)))

data=['1.서울돈화문국악당.csv',
 '10.창덕궁 앞 돈화문로.csv',
 '11.정동공원.csv',
 '12-1.종로 귀금속거리.csv',
 '12-2.낙원상가.csv',
 '13.돈의문 박물관 마을.csv',
 '14.서울로7017.csv',
 '15.세운상가.csv',
 '16.장안평 중고차 매매 시장.csv',
 '17.장안평 자동차 산업 통합 정보 센터.csv',
 '18.용산전자상가.csv',
 '19.청계천 박물관.csv',
 '2.정동1928아트센터.csv',
 '20.서울글로벌창업센터.csv',
 '21-1.팹랩 서울.csv',
 '21-2.용산 디지털 대장간.csv',
 '22.문화비축기지.csv',
 '23노들섬 노들서가.csv',
 '24.노들섬.csv',
 '25.성수 근린공원.csv',
 '26.서울책보고.csv',
 '27.성수 언더스텐드 에비뉴.csv',
 '28.가리봉 시장.csv',
 '29.창신숭인백남준기념관.csv',
 '3-1.대한성공회 서울주교좌성당.csv',
 '3-2.정동제일교회.csv',
 '3-3.약현성당.csv',
 '30.산마루 놀이터.csv',
 '31.해방촌 108계단.csv',
 '32.이음피움 봉제역사관.csv',
 '33.금천예술공장.csv',
 '34.한양도성 혜화동 전시안내 센터.csv',
 '35.해방촌 신흥시장.csv',
 '4-1.서울도시건축전시관.csv',
 '4-2.옛러시아공사관.csv',
 '5.덕수궁 중명전.csv',
 '6.국립극단.csv',
 '7.정동전망대.csv',
 '8.회현동 계단집.csv',
 '9.익선동 한옥거리.csv']

model = torch.load('model.pt')

model.eval()
transform = nlp.data.BERTSentenceTransform(tok,max_seq_length=64,pad=True,pair=False)


def evaluate(sentence):
    a,b,c = transform(sentence)

    a = torch.Tensor(a)
    b=torch.IntTensor([int(b)])
    c = torch.Tensor(c)

    a = a.reshape(1,64)
    c = c.reshape(1,c.shape[0])



    with torch.no_grad():

            a = a.long().to(device)
            c = c.long().to(device)     
    
            outputs = model(a,b,c)
        
    return np.argmax(outputs[0].detach().cpu().numpy())

scores=[]

for i in data:
    print(i)
    hi = pd.read_csv(i,encoding='utf-8')
    hi = hi.dropna()
    hi['caption'] = hi['caption'].apply(clean_text) 
    aa=list(hi.dropna()['caption'])
    sentiment=[]
    for j in aa:
        if(len(j)==0):
            continue
        sentiment.append(evaluate(j))
    score=0
    for i in sentiment:
        if i==1:
            score+=1
    scores.append(score/len(sentiment))

kk=pd.DataFrame(np.array(data))

kk['score'] = scores

kk.to_csv('y값.csv',encoding='utf-8-sig')

import pandas as pd
import re

hi = pd.read_csv('1.서울돈화문국악당.csv')

def clean_text(sentence):
    sentence=sentence.replace('(Google 번역 제공)','')
    sentence=sentence.replace('(원문)','')
    sentence=re.compile('[^ ㄱ-ㅣ가-힣]+').sub('',sentence)

    return sentence

hi=hi.dropna()
hi['caption'] = hi['caption'].apply(clean_text)

aa=list(hi.dropna()['caption'])

sentiment=[]

for i in aa:
    sentiment.append(evaluate(i))

kk=0

for i in sentiment:
    if i==1:
        kk+=1

kk/len(sentiment)

transform(['국악 공연장인데 깨끗하고 아늑한게 참 좋습니다. 공연 보러 2번째 방문인데 다음 공연이 기대됩니다. 영화나 대학로 연극과는 구별되는 색다른 경험을 느껴보세요.'])

tok('국악 공연장인데 깨끗하고 아늑한게 참 좋습니다. 공연 보러 2번째 방문인데 다음 공연이 기대됩니다. 영화나 대학로 연극과는 구별되는 색다른 경험을 느껴보세요.')


db8a6fe345678beefff991d63ce406f1f7825a6b
# -*- coding: utf-8 -*-
"""감성 분석

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1qQ50H_T1pDU6ZUANy_HFxxuddrLy_hvq
"""

import warnings
warnings.filterwarnings(action='ignore')

!pip install mxnet-cu101
!pip install gluonnlp padas tqdm
!pip install sentencepiece==0.1.85
!pip install transformers==2.1.1
!pip install torch==1.3.1

!pip install git+https://git@github.com/SKTBrain/KoBERT.git@master

!pip install gluonnlp

import torch
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import gluonnlp as nlp
import numpy as np
from tqdm import tqdm, tqdm_notebook

from kobert.utils import get_tokenizer
from kobert.pytorch_kobert import get_pytorch_kobert_model

from transformers import AdamW
from transformers.optimization import WarmupLinearSchedule

device = torch.device("cuda:0")

bertmodel, vocab = get_pytorch_kobert_model()

!wget https://www.dropbox.com/s/374ftkec978br3d/ratings_train.txt?dl=1
!wget https://www.dropbox.com/s/977gbwh542gdy94/ratings_test.txt?dl=1

dataset_train = nlp.data.TSVDataset("ratings_train.txt?dl=1", field_indices=[1,2], num_discard_samples=1)
dataset_test = nlp.data.TSVDataset("ratings_test.txt?dl=1", field_indices=[1,2], num_discard_samples=1)

tokenizer = get_tokenizer()
tok = nlp.data.BERTSPTokenizer(tokenizer,vocab,lower=False)

class BERTDataset(Dataset):
    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len, pad, pair):
        transform = nlp.data.BERTSentenceTransform(bert_tokenizer,max_seq_length=max_len,pad=pad,pair=pair)
        self.sentences = [transform([i[sent_idx]]) for i in dataset]
        self.labels = [np.int32(i[label_idx]) for i in dataset]

    def __getitem__(self,i):
        return (self.sentences[i] + (self.labels[i], ))

    def __len__(self):
        return (len(self.labels))

max_len = 64
batch_size = 64
warmup_ratio = 0.1
num_epochs = 5
max_grad_norm = 1
log_interval = 200
learning_rate = 5e-5

data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)
data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)

train_dataloader = torch.utils.data.DataLoader(data_train,batch_size=batch_size,num_workers=5)
test_dataloader = torch.utils.data.DataLoader(data_test,batch_size=batch_size,num_workers=5)

class BERTClassifier(nn.Module):
    def __init__(self,bert,hidden_size=768,num_classes=2,dr_rate=None,params=None):
        super(BERTClassifier,self).__init__()
        self.bert=bert
        self.dr_rate = dr_rate
        
        self.classifier = nn.Linear(hidden_size, num_classes)
        if dr_rate:
            self.dropout = nn.Dropout(p=dr_rate)

    def gen_attention_mask(self, token_ids, valid_length):
        attention_mask = torch.zeros_like(token_ids)
        for i,v in enumerate(valid_length):
            attention_mask[i][:v]=1
        return attention_mask.float()

    def forward(self, token_ids, valid_length, segment_ids):
        attention_mask = self.gen_attention_mask(token_ids,valid_length)

        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(),
                              attention_mask = attention_mask.float().to(token_ids.device))

        if self.dr_rate:
            out = self.dropout(pooler)
        
        return self.classifier(out)

model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)

# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]

optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
loss_fn = nn.CrossEntropyLoss()

t_total = len(train_dataloader) * num_epochs
warmup_step = int(t_total*warmup_ratio)

scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_step, t_total=t_total)

def calc_accuracy(X,Y):
    max_vals, max_indices = torch.max(X, 1)
    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]
    return train_acc

for e in range(num_epochs):
    train_acc=0.0
    test_acc=0.0
    model.train()

    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(train_dataloader)):
        optimizer.zero_grad()
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length = valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        loss = loss_fn(out, label)
        loss.backward()
        torch.nn.utils.clip_grad_norm(model.parameters(), max_grad_norm)
        optimizer.step()
        scheduler.step()

        if batch_id % log_interval == 0:
            print("epoch {} batch id {} loss {} train acc {}".format(e+1, batch_id+1, loss.data.cpu().numpy(), train_acc / (batch_id+1)))
    print("epoch {} train acc {}".format(e+1, train_acc / (batch_id+1)))
    model.eval()

    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(test_dataloader)):
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length= valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        test_acc += calc_accuracy(out, label)
    print("epoch {} test acc {}".format(e+1, test_acc / (batch_id+1)))

data=['1.서울돈화문국악당.csv',
 '10.창덕궁 앞 돈화문로.csv',
 '11.정동공원.csv',
 '12-1.종로 귀금속거리.csv',
 '12-2.낙원상가.csv',
 '13.돈의문 박물관 마을.csv',
 '14.서울로7017.csv',
 '15.세운상가.csv',
 '16.장안평 중고차 매매 시장.csv',
 '17.장안평 자동차 산업 통합 정보 센터.csv',
 '18.용산전자상가.csv',
 '19.청계천 박물관.csv',
 '2.정동1928아트센터.csv',
 '20.서울글로벌창업센터.csv',
 '21-1.팹랩 서울.csv',
 '21-2.용산 디지털 대장간.csv',
 '22.문화비축기지.csv',
 '23노들섬 노들서가.csv',
 '24.노들섬.csv',
 '25.성수 근린공원.csv',
 '26.서울책보고.csv',
 '27.성수 언더스텐드 에비뉴.csv',
 '28.가리봉 시장.csv',
 '29.창신숭인백남준기념관.csv',
 '3-1.대한성공회 서울주교좌성당.csv',
 '3-2.정동제일교회.csv',
 '3-3.약현성당.csv',
 '30.산마루 놀이터.csv',
 '31.해방촌 108계단.csv',
 '32.이음피움 봉제역사관.csv',
 '33.금천예술공장.csv',
 '34.한양도성 혜화동 전시안내 센터.csv',
 '35.해방촌 신흥시장.csv',
 '4-1.서울도시건축전시관.csv',
 '4-2.옛러시아공사관.csv',
 '5.덕수궁 중명전.csv',
 '6.국립극단.csv',
 '7.정동전망대.csv',
 '8.회현동 계단집.csv',
 '9.익선동 한옥거리.csv']

model = torch.load('model.pt')

model.eval()
transform = nlp.data.BERTSentenceTransform(tok,max_seq_length=64,pad=True,pair=False)


def evaluate(sentence):
    a,b,c = transform(sentence)

    a = torch.Tensor(a)
    b=torch.IntTensor([int(b)])
    c = torch.Tensor(c)

    a = a.reshape(1,64)
    c = c.reshape(1,c.shape[0])



    with torch.no_grad():

            a = a.long().to(device)
            c = c.long().to(device)     
    
            outputs = model(a,b,c)
        
    return np.argmax(outputs[0].detach().cpu().numpy())

scores=[]

for i in data:
    print(i)
    hi = pd.read_csv(i,encoding='utf-8')
    hi = hi.dropna()
    hi['caption'] = hi['caption'].apply(clean_text) 
    aa=list(hi.dropna()['caption'])
    sentiment=[]
    for j in aa:
        if(len(j)==0):
            continue
        sentiment.append(evaluate(j))
    score=0
    for i in sentiment:
        if i==1:
            score+=1
    scores.append(score/len(sentiment))

kk=pd.DataFrame(np.array(data))

kk['score'] = scores

kk.to_csv('y값.csv',encoding='utf-8-sig')

import pandas as pd
import re

hi = pd.read_csv('1.서울돈화문국악당.csv')

def clean_text(sentence):
    sentence=sentence.replace('(Google 번역 제공)','')
    sentence=sentence.replace('(원문)','')
    sentence=re.compile('[^ ㄱ-ㅣ가-힣]+').sub('',sentence)

    return sentence

hi=hi.dropna()
hi['caption'] = hi['caption'].apply(clean_text)

aa=list(hi.dropna()['caption'])

sentiment=[]

for i in aa:
    sentiment.append(evaluate(i))

kk=0

for i in sentiment:
    if i==1:
        kk+=1

kk/len(sentiment)

transform(['국악 공연장인데 깨끗하고 아늑한게 참 좋습니다. 공연 보러 2번째 방문인데 다음 공연이 기대됩니다. 영화나 대학로 연극과는 구별되는 색다른 경험을 느껴보세요.'])

tok('국악 공연장인데 깨끗하고 아늑한게 참 좋습니다. 공연 보러 2번째 방문인데 다음 공연이 기대됩니다. 영화나 대학로 연극과는 구별되는 색다른 경험을 느껴보세요.')

