import pandas as pd
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import train_test_split
import time
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB


# Dataset Loading
df = pd.read_csv("/archive/Dataset/Dataset/train/LargeTrain.csv")


X = df.iloc[:,:-1]
Y = df.iloc[:,-1]

# Class Labels are from 1 to 9 (type str and int )
X = X.astype(float)
# Convert Class labels into only integers
Y = Y.astype(int)
# Train Test spliting
# X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=10, test_size=0.2)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, shuffle = True, test_size=0.1)

# class labels value counts
Y.value_counts()
# =============================================================================
# KNN for n_neighbors
# =============================================================================
print('-'*25,'KNN','-'*25)
from sklearn import neighbors
start = time.time()
print('program start...')
print()

# Normalization 
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
scaler.fit(X_test)
X_test = scaler.transform(X_test)
normalized_y = Y

# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, normalized_y, shuffle = True, test_size=0.2)


KNN_model=neighbors.KNeighborsClassifier(n_neighbors=3,n_jobs=-1)
KNN_model.fit(X_train,Y_train)
pred=KNN_model.predict(X_test)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print("Accuracy={}%".format((sum(Y_test==pred)/Y_test.shape[0])*100))
confusion_mat = confusion_matrix(Y_test.to_numpy(), pred)
print(confusion_matrix)
print(classification_report(Y_test, pred))
print('#'*70)


# =============================================================================
# Logistic Regression
# =============================================================================

print('-'*25,"logistic",'-'*25)
start = time.time()
print('program start...')
from pandas import Series, DataFrame
from pylab import rcParams
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
# from sklearn.cross_validation import train_test_split
from sklearn import metrics 
from sklearn.metrics import classification_report

print()


LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)

y_pred = LogReg.predict(X_test)
print("Accuracy={}%".format((sum(Y_test==y_pred)/Y_test.shape[0])*100))
# from sklearn.metrics import confusion_matrix
# confusion_matrix = confusion_matrix(Y_test, y_pred)
print(classification_report(Y_test, y_pred))
print('#'*70)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# Decision Tree
# =============================================================================
DT = DecisionTreeClassifier()
print('-'*35,'DT','-'*35)
start = time.time()
print('program start...')
print()

DT.fit(X_train, Y_train)
print()

print('prediction:')
y_pred = DT.predict(X_test)
print(y_pred)
print()

print('Score:')
score = DT.score(X_test,Y_test)
print(score)

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

print("Classifiction Report :")
print(classification_report(Y_test, y_pred))
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, y_pred, labels=labels)
print(cm)
print('#'*70)

# =============================================================================
# Naive Bayes GaussianNB
# =============================================================================
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB
print()
print()
print('-'*25,'GaussianNB','-'*25)

# from sklearn.preprocessing import MinMaxScaler
# scaler = MinMaxScaler()
# scaler.fit(X)
# normalized_x = scaler.transform(X)



# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle = True, test_size=0.25)


clf = GaussianNB()
clf.fit(X_train, Y_train)


start = time.time()
print('program start...')
print()

clf = GaussianNB().fit(X_train, Y_train)
print()
print(clf.score(X_test, Y_test))
print()

y_pred = clf.fit(X_train, Y_train).predict(X_test)
print(y_pred)
print()

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

########### Cross Fold for GaussianNB ################

print("Cross Validation Accuracy: \n")

start = time.time()
print('program start...')
print()
cv_accuracy = cross_val_score(estimator=clf, X=X_train, y=Y_train, cv=10)
print("Accuracy using 10 folds: ")
print(cv_accuracy)

print("\n")

print("Mean accuracy: {}".format(cv_accuracy.mean()))
print("Standard Deviation: {}".format(cv_accuracy.std()))

print("\n")

print("Confusion Matrix for Naive Bayes GaussianNB\n")
predicted = clf.predict(X_test)
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)

print("Classifiction Report :")
print(classification_report(Y_test, y_pred))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# Multinomial Naive Bayes with cross fold
# =============================================================================
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.preprocessing import MinMaxScaler

start = time.time()
print('program start...')
print()
# scaler = MinMaxScaler()
# scaler.fit(X)
# normalized_x = scaler.transform(X)
# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle=True, test_size=0.25)

model = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
model.fit(X_train, Y_train)

# Print results to evaluate model
print("Showing Performance Metrics for Naive Bayes Multinomial\n")
print ("Training Accuracy: {}".format(model.score(X_train, Y_train)))
predicted = model.predict(X_test)
print ("Testing Accuracy: {}".format(accuracy_score(Y_test, predicted)))

print("\n")

print("Cross Validation Accuracy: \n")
cv_accuracy = cross_val_score(estimator=model, X=X_train, y=Y_train, cv=10)
print("Accuracy using 10 folds: ")
print(cv_accuracy)

print("\n")

print("Mean accuracy: {}".format(cv_accuracy.mean()))
print("Standard Deviation: {}".format(cv_accuracy.std()))

print("\n")

print("Confusion Matrix for Naive Bayes Multinomial\n")
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)

print("\n")

print('Precision, Recall and f-1 Scores for Naive Bayes Multinomial\n')
print(classification_report(Y_test, predicted))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# 
# =============================================================================


# =============================================================================
# SVM
# =============================================================================
print()
print()
print('-'*35,'SVM','-'*35)
SVM_classifier = SVC()


# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle=True, test_size=0.2)
start = time.time()
print('program start...')
print()

SVM_classifier = SVC(C=1.0, cache_size=1500, verbose=True).fit(X_train, Y_train)
print()
print(SVM_classifier.score(X_test, Y_test))
print()

y_pred = SVM_classifier.predict(X_test)
print(y_pred)
print()

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print("Classifiction Report :")
print(classification_report(Y_test, y_pred,labels=np.unique(y_pred)))

# =============================================================================
# SVM Tuning 
# =============================================================================
model = SVC()
start = time.time()
print('program start...')
print()


param_grid = [{'kernel': ['rbf'],
               'gamma': [1e-4, 0.01, 0.1],
               'C': [0.01, 1]}]

print("Hyper Parameter Tuning Results\n")

# Finding optimum parameters through GridSearchCV
grid = GridSearchCV(estimator=model, param_grid = param_grid,
                    cv = 5)
grid.fit(X_train, Y_train)

print("\n")
print("Results returned by GridSearchCV\n")
print("Best estimator: ", grid.best_estimator_)
print("\n")
print("Best score: ", grid.best_score_)
print("\n")
print("Best parameters found: ", grid.best_params_)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')


# =============================================================================
# KNN - less frequent class 5,8 drop 
# =============================================================================
print('-'*25,'KNN Modified','-'*25)
dff = df[df.iloc[:,-1] != 5]
dff = dff[dff.iloc[:,-1] != 8]
dff.iloc[:,-1].value_counts()
X = dff.iloc[:,:-1]
# X2 = df2.iloc[:,:-1]
Y = dff.iloc[:,-1]
# Y2 = df2.iloc[:,-1]

X = X.astype(float)
# X2 = X2.astype(float)
Y = Y.astype(int)
# Y2 = Y2.astype(int)

scaler = MinMaxScaler()
scaler.fit(X)
normalized_x = scaler.transform(X)
normalized_y = Y

X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, normalized_y, random_state=10, test_size=0.2)
KNN_model=neighbors.KNeighborsClassifier(n_neighbors=3,n_jobs=-1)
KNN_model.fit(X_train,Y_train)
pred=KNN_model.predict(X_test)
print("Accuracy={}%".format((sum(Y_test==pred)/Y_test.shape[0])*100))
print(classification_report(Y_test, pred))
print('#'*70)
692dafb82d21b0eb06fb390b39e55f48c2d0ba4a
<mask token>

<mask token>
Y.value_counts()
print('-' * 25, 'KNN', '-' * 25)
<mask token>
print('program start...')
print()
<mask token>
scaler.fit(X_train)
<mask token>
scaler.fit(X_test)
<mask token>
KNN_model.fit(X_train, Y_train)
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
<mask token>
print(confusion_matrix)
print(classification_report(Y_test, pred))
print('#' * 70)
print('-' * 25, 'logistic', '-' * 25)
<mask token>
print('program start...')
<mask token>
print()
<mask token>
LogReg.fit(X_train, Y_train)
<mask token>
print('Accuracy={}%'.format(sum(Y_test == y_pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, y_pred))
print('#' * 70)
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
<mask token>
print('-' * 35, 'DT', '-' * 35)
<mask token>
print('program start...')
print()
DT.fit(X_train, Y_train)
print()
print('prediction:')
<mask token>
print(y_pred)
print()
print('Score:')
<mask token>
print(score)
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
<mask token>
print(cm)
print('#' * 70)
<mask token>
print()
print()
print('-' * 25, 'GaussianNB', '-' * 25)
<mask token>
clf.fit(X_train, Y_train)
<mask token>
print('program start...')
print()
<mask token>
print()
print(clf.score(X_test, Y_test))
print()
<mask token>
print(y_pred)
print()
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Cross Validation Accuracy: \n')
<mask token>
print('program start...')
print()
<mask token>
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes GaussianNB\n')
<mask token>
print(cm)
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
<mask token>
print('program start...')
print()
<mask token>
model.fit(X_train, Y_train)
print('Showing Performance Metrics for Naive Bayes Multinomial\n')
print('Training Accuracy: {}'.format(model.score(X_train, Y_train)))
<mask token>
print('Testing Accuracy: {}'.format(accuracy_score(Y_test, predicted)))
print('\n')
print('Cross Validation Accuracy: \n')
<mask token>
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes Multinomial\n')
<mask token>
print(cm)
print('\n')
print('Precision, Recall and f-1 Scores for Naive Bayes Multinomial\n')
print(classification_report(Y_test, predicted))
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print()
print()
print('-' * 35, 'SVM', '-' * 35)
<mask token>
print('program start...')
print()
<mask token>
print()
print(SVM_classifier.score(X_test, Y_test))
print()
<mask token>
print(y_pred)
print()
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred, labels=np.unique(y_pred)))
<mask token>
print('program start...')
print()
<mask token>
print('Hyper Parameter Tuning Results\n')
<mask token>
grid.fit(X_train, Y_train)
print('\n')
print('Results returned by GridSearchCV\n')
print('Best estimator: ', grid.best_estimator_)
print('\n')
print('Best score: ', grid.best_score_)
print('\n')
print('Best parameters found: ', grid.best_params_)
<mask token>
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('-' * 25, 'KNN Modified', '-' * 25)
<mask token>
dff.iloc[:, -1].value_counts()
<mask token>
scaler.fit(X)
<mask token>
KNN_model.fit(X_train, Y_train)
<mask token>
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, pred))
print('#' * 70)

<mask token>
df = pd.read_csv('/archive/Dataset/Dataset/train/LargeTrain.csv')
X = df.iloc[:, :-1]
Y = df.iloc[:, -1]
X = X.astype(float)
Y = Y.astype(int)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, shuffle=True,
    test_size=0.1)
Y.value_counts()
print('-' * 25, 'KNN', '-' * 25)
<mask token>
start = time.time()
print('program start...')
print()
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
scaler.fit(X_test)
X_test = scaler.transform(X_test)
normalized_y = Y
KNN_model = neighbors.KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
KNN_model.fit(X_train, Y_train)
pred = KNN_model.predict(X_test)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
confusion_mat = confusion_matrix(Y_test.to_numpy(), pred)
print(confusion_matrix)
print(classification_report(Y_test, pred))
print('#' * 70)
print('-' * 25, 'logistic', '-' * 25)
start = time.time()
print('program start...')
<mask token>
print()
LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)
y_pred = LogReg.predict(X_test)
print('Accuracy={}%'.format(sum(Y_test == y_pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, y_pred))
print('#' * 70)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
DT = DecisionTreeClassifier()
print('-' * 35, 'DT', '-' * 35)
start = time.time()
print('program start...')
print()
DT.fit(X_train, Y_train)
print()
print('prediction:')
y_pred = DT.predict(X_test)
print(y_pred)
print()
print('Score:')
score = DT.score(X_test, Y_test)
print(score)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, y_pred, labels=labels)
print(cm)
print('#' * 70)
<mask token>
print()
print()
print('-' * 25, 'GaussianNB', '-' * 25)
clf = GaussianNB()
clf.fit(X_train, Y_train)
start = time.time()
print('program start...')
print()
clf = GaussianNB().fit(X_train, Y_train)
print()
print(clf.score(X_test, Y_test))
print()
y_pred = clf.fit(X_train, Y_train).predict(X_test)
print(y_pred)
print()
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Cross Validation Accuracy: \n')
start = time.time()
print('program start...')
print()
cv_accuracy = cross_val_score(estimator=clf, X=X_train, y=Y_train, cv=10)
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes GaussianNB\n')
predicted = clf.predict(X_test)
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
<mask token>
start = time.time()
print('program start...')
print()
model = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
model.fit(X_train, Y_train)
print('Showing Performance Metrics for Naive Bayes Multinomial\n')
print('Training Accuracy: {}'.format(model.score(X_train, Y_train)))
predicted = model.predict(X_test)
print('Testing Accuracy: {}'.format(accuracy_score(Y_test, predicted)))
print('\n')
print('Cross Validation Accuracy: \n')
cv_accuracy = cross_val_score(estimator=model, X=X_train, y=Y_train, cv=10)
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes Multinomial\n')
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)
print('\n')
print('Precision, Recall and f-1 Scores for Naive Bayes Multinomial\n')
print(classification_report(Y_test, predicted))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print()
print()
print('-' * 35, 'SVM', '-' * 35)
SVM_classifier = SVC()
start = time.time()
print('program start...')
print()
SVM_classifier = SVC(C=1.0, cache_size=1500, verbose=True).fit(X_train, Y_train
    )
print()
print(SVM_classifier.score(X_test, Y_test))
print()
y_pred = SVM_classifier.predict(X_test)
print(y_pred)
print()
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred, labels=np.unique(y_pred)))
model = SVC()
start = time.time()
print('program start...')
print()
param_grid = [{'kernel': ['rbf'], 'gamma': [0.0001, 0.01, 0.1], 'C': [0.01, 1]}
    ]
print('Hyper Parameter Tuning Results\n')
grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=5)
grid.fit(X_train, Y_train)
print('\n')
print('Results returned by GridSearchCV\n')
print('Best estimator: ', grid.best_estimator_)
print('\n')
print('Best score: ', grid.best_score_)
print('\n')
print('Best parameters found: ', grid.best_params_)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('-' * 25, 'KNN Modified', '-' * 25)
dff = df[df.iloc[:, -1] != 5]
dff = dff[dff.iloc[:, -1] != 8]
dff.iloc[:, -1].value_counts()
X = dff.iloc[:, :-1]
Y = dff.iloc[:, -1]
X = X.astype(float)
Y = Y.astype(int)
scaler = MinMaxScaler()
scaler.fit(X)
normalized_x = scaler.transform(X)
normalized_y = Y
X_train, X_test, Y_train, Y_test = train_test_split(normalized_x,
    normalized_y, random_state=10, test_size=0.2)
KNN_model = neighbors.KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
KNN_model.fit(X_train, Y_train)
pred = KNN_model.predict(X_test)
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, pred))
print('#' * 70)

import pandas as pd
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import train_test_split
import time
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB
df = pd.read_csv('/archive/Dataset/Dataset/train/LargeTrain.csv')
X = df.iloc[:, :-1]
Y = df.iloc[:, -1]
X = X.astype(float)
Y = Y.astype(int)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, shuffle=True,
    test_size=0.1)
Y.value_counts()
print('-' * 25, 'KNN', '-' * 25)
from sklearn import neighbors
start = time.time()
print('program start...')
print()
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
scaler.fit(X_test)
X_test = scaler.transform(X_test)
normalized_y = Y
KNN_model = neighbors.KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
KNN_model.fit(X_train, Y_train)
pred = KNN_model.predict(X_test)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
confusion_mat = confusion_matrix(Y_test.to_numpy(), pred)
print(confusion_matrix)
print(classification_report(Y_test, pred))
print('#' * 70)
print('-' * 25, 'logistic', '-' * 25)
start = time.time()
print('program start...')
from pandas import Series, DataFrame
from pylab import rcParams
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import classification_report
print()
LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)
y_pred = LogReg.predict(X_test)
print('Accuracy={}%'.format(sum(Y_test == y_pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, y_pred))
print('#' * 70)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
DT = DecisionTreeClassifier()
print('-' * 35, 'DT', '-' * 35)
start = time.time()
print('program start...')
print()
DT.fit(X_train, Y_train)
print()
print('prediction:')
y_pred = DT.predict(X_test)
print(y_pred)
print()
print('Score:')
score = DT.score(X_test, Y_test)
print(score)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, y_pred, labels=labels)
print(cm)
print('#' * 70)
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB
print()
print()
print('-' * 25, 'GaussianNB', '-' * 25)
clf = GaussianNB()
clf.fit(X_train, Y_train)
start = time.time()
print('program start...')
print()
clf = GaussianNB().fit(X_train, Y_train)
print()
print(clf.score(X_test, Y_test))
print()
y_pred = clf.fit(X_train, Y_train).predict(X_test)
print(y_pred)
print()
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Cross Validation Accuracy: \n')
start = time.time()
print('program start...')
print()
cv_accuracy = cross_val_score(estimator=clf, X=X_train, y=Y_train, cv=10)
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes GaussianNB\n')
predicted = clf.predict(X_test)
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)
print('Classifiction Report :')
print(classification_report(Y_test, y_pred))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.preprocessing import MinMaxScaler
start = time.time()
print('program start...')
print()
model = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
model.fit(X_train, Y_train)
print('Showing Performance Metrics for Naive Bayes Multinomial\n')
print('Training Accuracy: {}'.format(model.score(X_train, Y_train)))
predicted = model.predict(X_test)
print('Testing Accuracy: {}'.format(accuracy_score(Y_test, predicted)))
print('\n')
print('Cross Validation Accuracy: \n')
cv_accuracy = cross_val_score(estimator=model, X=X_train, y=Y_train, cv=10)
print('Accuracy using 10 folds: ')
print(cv_accuracy)
print('\n')
print('Mean accuracy: {}'.format(cv_accuracy.mean()))
print('Standard Deviation: {}'.format(cv_accuracy.std()))
print('\n')
print('Confusion Matrix for Naive Bayes Multinomial\n')
labels = [x for x in range(1, 10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)
print('\n')
print('Precision, Recall and f-1 Scores for Naive Bayes Multinomial\n')
print(classification_report(Y_test, predicted))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print()
print()
print('-' * 35, 'SVM', '-' * 35)
SVM_classifier = SVC()
start = time.time()
print('program start...')
print()
SVM_classifier = SVC(C=1.0, cache_size=1500, verbose=True).fit(X_train, Y_train
    )
print()
print(SVM_classifier.score(X_test, Y_test))
print()
y_pred = SVM_classifier.predict(X_test)
print(y_pred)
print()
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('Classifiction Report :')
print(classification_report(Y_test, y_pred, labels=np.unique(y_pred)))
model = SVC()
start = time.time()
print('program start...')
print()
param_grid = [{'kernel': ['rbf'], 'gamma': [0.0001, 0.01, 0.1], 'C': [0.01, 1]}
    ]
print('Hyper Parameter Tuning Results\n')
grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=5)
grid.fit(X_train, Y_train)
print('\n')
print('Results returned by GridSearchCV\n')
print('Best estimator: ', grid.best_estimator_)
print('\n')
print('Best score: ', grid.best_score_)
print('\n')
print('Best parameters found: ', grid.best_params_)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print('-' * 25, 'KNN Modified', '-' * 25)
dff = df[df.iloc[:, -1] != 5]
dff = dff[dff.iloc[:, -1] != 8]
dff.iloc[:, -1].value_counts()
X = dff.iloc[:, :-1]
Y = dff.iloc[:, -1]
X = X.astype(float)
Y = Y.astype(int)
scaler = MinMaxScaler()
scaler.fit(X)
normalized_x = scaler.transform(X)
normalized_y = Y
X_train, X_test, Y_train, Y_test = train_test_split(normalized_x,
    normalized_y, random_state=10, test_size=0.2)
KNN_model = neighbors.KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
KNN_model.fit(X_train, Y_train)
pred = KNN_model.predict(X_test)
print('Accuracy={}%'.format(sum(Y_test == pred) / Y_test.shape[0] * 100))
print(classification_report(Y_test, pred))
print('#' * 70)

import pandas as pd
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import train_test_split
import time
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB


# Dataset Loading
df = pd.read_csv("/archive/Dataset/Dataset/train/LargeTrain.csv")


X = df.iloc[:,:-1]
Y = df.iloc[:,-1]

# Class Labels are from 1 to 9 (type str and int )
X = X.astype(float)
# Convert Class labels into only integers
Y = Y.astype(int)
# Train Test spliting
# X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=10, test_size=0.2)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, shuffle = True, test_size=0.1)

# class labels value counts
Y.value_counts()
# =============================================================================
# KNN for n_neighbors
# =============================================================================
print('-'*25,'KNN','-'*25)
from sklearn import neighbors
start = time.time()
print('program start...')
print()

# Normalization 
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
scaler.fit(X_test)
X_test = scaler.transform(X_test)
normalized_y = Y

# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, normalized_y, shuffle = True, test_size=0.2)


KNN_model=neighbors.KNeighborsClassifier(n_neighbors=3,n_jobs=-1)
KNN_model.fit(X_train,Y_train)
pred=KNN_model.predict(X_test)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print("Accuracy={}%".format((sum(Y_test==pred)/Y_test.shape[0])*100))
confusion_mat = confusion_matrix(Y_test.to_numpy(), pred)
print(confusion_matrix)
print(classification_report(Y_test, pred))
print('#'*70)


# =============================================================================
# Logistic Regression
# =============================================================================

print('-'*25,"logistic",'-'*25)
start = time.time()
print('program start...')
from pandas import Series, DataFrame
from pylab import rcParams
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
# from sklearn.cross_validation import train_test_split
from sklearn import metrics 
from sklearn.metrics import classification_report

print()


LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)

y_pred = LogReg.predict(X_test)
print("Accuracy={}%".format((sum(Y_test==y_pred)/Y_test.shape[0])*100))
# from sklearn.metrics import confusion_matrix
# confusion_matrix = confusion_matrix(Y_test, y_pred)
print(classification_report(Y_test, y_pred))
print('#'*70)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# Decision Tree
# =============================================================================
DT = DecisionTreeClassifier()
print('-'*35,'DT','-'*35)
start = time.time()
print('program start...')
print()

DT.fit(X_train, Y_train)
print()

print('prediction:')
y_pred = DT.predict(X_test)
print(y_pred)
print()

print('Score:')
score = DT.score(X_test,Y_test)
print(score)

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

print("Classifiction Report :")
print(classification_report(Y_test, y_pred))
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, y_pred, labels=labels)
print(cm)
print('#'*70)

# =============================================================================
# Naive Bayes GaussianNB
# =============================================================================
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.naive_bayes import GaussianNB
print()
print()
print('-'*25,'GaussianNB','-'*25)

# from sklearn.preprocessing import MinMaxScaler
# scaler = MinMaxScaler()
# scaler.fit(X)
# normalized_x = scaler.transform(X)



# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle = True, test_size=0.25)


clf = GaussianNB()
clf.fit(X_train, Y_train)


start = time.time()
print('program start...')
print()

clf = GaussianNB().fit(X_train, Y_train)
print()
print(clf.score(X_test, Y_test))
print()

y_pred = clf.fit(X_train, Y_train).predict(X_test)
print(y_pred)
print()

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

########### Cross Fold for GaussianNB ################

print("Cross Validation Accuracy: \n")

start = time.time()
print('program start...')
print()
cv_accuracy = cross_val_score(estimator=clf, X=X_train, y=Y_train, cv=10)
print("Accuracy using 10 folds: ")
print(cv_accuracy)

print("\n")

print("Mean accuracy: {}".format(cv_accuracy.mean()))
print("Standard Deviation: {}".format(cv_accuracy.std()))

print("\n")

print("Confusion Matrix for Naive Bayes GaussianNB\n")
predicted = clf.predict(X_test)
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)

print("Classifiction Report :")
print(classification_report(Y_test, y_pred))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# Multinomial Naive Bayes with cross fold
# =============================================================================
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.preprocessing import MinMaxScaler

start = time.time()
print('program start...')
print()
# scaler = MinMaxScaler()
# scaler.fit(X)
# normalized_x = scaler.transform(X)
# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle=True, test_size=0.25)

model = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
model.fit(X_train, Y_train)

# Print results to evaluate model
print("Showing Performance Metrics for Naive Bayes Multinomial\n")
print ("Training Accuracy: {}".format(model.score(X_train, Y_train)))
predicted = model.predict(X_test)
print ("Testing Accuracy: {}".format(accuracy_score(Y_test, predicted)))

print("\n")

print("Cross Validation Accuracy: \n")
cv_accuracy = cross_val_score(estimator=model, X=X_train, y=Y_train, cv=10)
print("Accuracy using 10 folds: ")
print(cv_accuracy)

print("\n")

print("Mean accuracy: {}".format(cv_accuracy.mean()))
print("Standard Deviation: {}".format(cv_accuracy.std()))

print("\n")

print("Confusion Matrix for Naive Bayes Multinomial\n")
labels = [x for x in range(1,10)]
cm = confusion_matrix(Y_test, predicted, labels=labels)
print(cm)

print("\n")

print('Precision, Recall and f-1 Scores for Naive Bayes Multinomial\n')
print(classification_report(Y_test, predicted))
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')

# =============================================================================
# 
# =============================================================================


# =============================================================================
# SVM
# =============================================================================
print()
print()
print('-'*35,'SVM','-'*35)
SVM_classifier = SVC()


# X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, Y, shuffle=True, test_size=0.2)
start = time.time()
print('program start...')
print()

SVM_classifier = SVC(C=1.0, cache_size=1500, verbose=True).fit(X_train, Y_train)
print()
print(SVM_classifier.score(X_test, Y_test))
print()

y_pred = SVM_classifier.predict(X_test)
print(y_pred)
print()

end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')
print("Classifiction Report :")
print(classification_report(Y_test, y_pred,labels=np.unique(y_pred)))

# =============================================================================
# SVM Tuning 
# =============================================================================
model = SVC()
start = time.time()
print('program start...')
print()


param_grid = [{'kernel': ['rbf'],
               'gamma': [1e-4, 0.01, 0.1],
               'C': [0.01, 1]}]

print("Hyper Parameter Tuning Results\n")

# Finding optimum parameters through GridSearchCV
grid = GridSearchCV(estimator=model, param_grid = param_grid,
                    cv = 5)
grid.fit(X_train, Y_train)

print("\n")
print("Results returned by GridSearchCV\n")
print("Best estimator: ", grid.best_estimator_)
print("\n")
print("Best score: ", grid.best_score_)
print("\n")
print("Best parameters found: ", grid.best_params_)
end = time.time()
print('program end...')
print()
print('time cost: ')
print(end - start, 'seconds')


# =============================================================================
# KNN - less frequent class 5,8 drop 
# =============================================================================
print('-'*25,'KNN Modified','-'*25)
dff = df[df.iloc[:,-1] != 5]
dff = dff[dff.iloc[:,-1] != 8]
dff.iloc[:,-1].value_counts()
X = dff.iloc[:,:-1]
# X2 = df2.iloc[:,:-1]
Y = dff.iloc[:,-1]
# Y2 = df2.iloc[:,-1]

X = X.astype(float)
# X2 = X2.astype(float)
Y = Y.astype(int)
# Y2 = Y2.astype(int)

scaler = MinMaxScaler()
scaler.fit(X)
normalized_x = scaler.transform(X)
normalized_y = Y

X_train, X_test, Y_train, Y_test = train_test_split(normalized_x, normalized_y, random_state=10, test_size=0.2)
KNN_model=neighbors.KNeighborsClassifier(n_neighbors=3,n_jobs=-1)
KNN_model.fit(X_train,Y_train)
pred=KNN_model.predict(X_test)
print("Accuracy={}%".format((sum(Y_test==pred)/Y_test.shape[0])*100))
print(classification_report(Y_test, pred))
print('#'*70)