#!/usr/bin/env python
# coding: utf-8

# In[2]:


import pandas as pd
import numpy as np


# In[3]:


train = pd.read_csv(r"C:\Users\Mrinal M\Desktop\ML_assignment\Malware_train.csv")
test = pd.read_csv(r"C:\Users\Mrinal M\Desktop\ML_assignment\Malware_test.csv")
remove_cols = ['Census_InternalBatteryType','DefaultBrowsersIdentifier','PuaMode', 'Census_ProcessorClass', 'Census_IsWIMBootEnabled', 'IsBeta', 'Census_IsFlightsDisabled', 'Census_IsFlightingInternal', 'AutoSampleOptIn', 'Census_ThresholdOptIn', 'SMode', 'Census_IsPortableOperatingSystem', 'Census_DeviceFamily', 'UacLuaenable', 'Census_IsVirtualDevice', 'Platform', 'Census_OSSkuName', 'Census_OSInstallLanguageIdentifier', 'Processor']
# The columns in remove_Cols are columns with 90percent nan values and heavy biased columns.
train.drop(remove_cols, axis=1, inplace=True)
id_ = test['MachineIdentifier']
test.drop(remove_cols, axis=1, inplace=True)


# In[4]:


stats = []
for col in train.columns:
    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 100 / train.shape[0], train[col].value_counts(normalize=True, dropna=False).values[0] * 100, train[col].dtype))
    
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])
stats_df.sort_values('Percentage of missing values', ascending=False)


# In[5]:


label = []
for col in train.columns:
    if train[col].dtypes == object:
        unique = len(train[col].unique())
        if unique>50:
            label.append(col)
        print("Feature {} has {} unique categories".format(col,unique))
print(label)
def handle_non_numerical_data(df,columns):
#     columns = df.columns.values
    for column in columns:
        text_digit_vals = {}
        def convert_to_int(val):
            return text_digit_vals[val]

        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x+=1

            df[column] = list(map(convert_to_int, df[column]))

    return df
# train = handle_non_numerical_data(train)
# test = handle_non_numerical_data(test)
train = handle_non_numerical_data(train,label)
test = handle_non_numerical_data(test,label)
# # label = ['MachineIdentifier','AvSigVersion','AVProductStatesIdentifier','CityIdentifier','Census_OEMNameIdentifier','Census_OEMModelIdentifier','Census_ProcessorModelIdentifier','Census_PrimaryDiskTotalCapacity','Census_InternalBatteryNumberOfCharges',]


# In[6]:


l = ['mse' ,'windowsintune', 'mseprerelease']
train['ProductName'] = [1 if i in l else 0 for i in train['ProductName']]
test['ProductName'] = [1 if i in l else 0 for i in test['ProductName']]
dummies = pd.get_dummies(train['ProductName'], prefix='ProductName',dummy_na=False)
train = train.drop('ProductName',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['ProductName'], prefix='ProductName',dummy_na=False)
test = test.drop('ProductName',1)
test = pd.concat([test,dummies],axis=1)
l = ['6.1.1.0' ,'6.3.0.0', '6.1.0.0', '10.0.1.0' ,'6.3.1.0','10.0.32.72', '10.0.80.0', '10.0.0.2']
train['OsVer'] = [1 if i in l else 0 for i in train['OsVer']]
test['OsVer'] = [1 if i in l else 0 for i in test['OsVer']]
dummies = pd.get_dummies(train['OsVer'], prefix='OsVer',dummy_na=False)
train = train.drop('OsVer',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['OsVer'], prefix='OsVer',dummy_na=False)
test = test.drop('OsVer',1)
test = pd.concat([test,dummies],axis=1)
dummies = pd.get_dummies(train['OsPlatformSubRelease'], prefix='OsPlatformSubRelease',dummy_na=False)
train = train.drop('OsPlatformSubRelease',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['OsPlatformSubRelease'], prefix='OsPlatformSubRelease',dummy_na=False)
test = test.drop('OsPlatformSubRelease',1)
test = pd.concat([test,dummies],axis=1)
home = ['Home']
pro = ['Pro']
other = ['Education','Enterprise LTSB','Enterprise','Invalid','Cloud','Server']
SkuEdition = []
for i in train['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
train['SkuEdition'] = SkuEdition
SkuEdition = []
for i in test['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
test['SkuEdition'] = SkuEdition
dummies = pd.get_dummies(train['SkuEdition'], prefix='SkuEdition',dummy_na=False)
train = train.drop('SkuEdition',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['SkuEdition'], prefix='SkuEdition',dummy_na=False)
test = test.drop('SkuEdition',1)
test = pd.concat([test,dummies],axis=1)
home = ['RequireAdmin']
pro = ['ExistsNotSet']
Ss = []
for i in train['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train['SmartScreen'] = Ss
Ss = []
for i in test['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test['SmartScreen'] = Ss
dummies = pd.get_dummies(train['SmartScreen'], prefix='SmartScreen',dummy_na=False)
train = train.drop('SmartScreen',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['SmartScreen'], prefix='SmartScreen',dummy_na=False)
test = test.drop('SmartScreen',1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_MDC2FormFactor'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_PrimaryDiskTypeName'  
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_PowerPlatformRoleName'
home = ['Desktop']
pro = ['Mobile']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
col = 'Census_PowerPlatformRoleName'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSArchitecture'
train[col] = [1 if i in l else 0 for i in train[col]]
test[col] = [1 if i in l else 0 for i in test[col]]
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_FlightRing'
home = ['Retail']
pro = ['NOT_SET']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_ActivationChannel'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_GenuineStateName'
home = ['IS_GENUINE']
pro = ['INVALID_LICENSE']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)

col = 'Census_OSWUAutoUpdateOptionsName'
home = ['FullAuto']
pro = ['Notify']
blah = ['AutoInstallAndRebootAtMaintenanceTime']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else: Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSInstallTypeName'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSEdition'
home = ['Professional']
pro = ['CoreSingleLanguage']
blah = ['Core']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else: Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSBranch'
home = ['rs4_release']
pro = ['rs3_release']
blah = ['rs2_release']
haha = ['rs1_release']
hehe = ['rs3_release_svc_escrow']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_ChassisTypeName'
home = ['Notebook']
pro = ['Desktop']
blah = ['Laptop']
haha = ['Portable']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)


# In[7]:


from sklearn.impute import SimpleImputer
imp = SimpleImputer(missing_values=np.NaN, strategy = 'median')
train = pd.DataFrame(imp.fit_transform(train),columns=train.columns)
test = pd.DataFrame(imp.fit_transform(test),columns=test.columns)


# In[6]:


# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# from imblearn.under_sampling import RandomUnderSampler
# rus = RandomUnderSampler()
# X_rus, y_rus= rus.fit_sample(X_train, y_train)


# In[28]:


# from imblearn.over_sampling import SMOTE
# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# smote = SMOTE(sampling_strategy='minority')
# X_sm, y_sm = smote.fit_sample(X_train, y_train)


# In[15]:


# # Class count
# count_class_0, count_class_1 = train.HasDetections.value_counts()

# # Divide by class
# df_class_0 = train[train['HasDetections'] == 0]
# df_class_1 = train[train['HasDetections'] == 1]


# In[16]:


# df_class_0_under = df_class_0.sample(count_class_1)
# train = pd.concat([df_class_0_under, df_class_1], axis=0)

# print('Random under-sampling:')
# print(train.HasDetections.value_counts())

# train.HasDetections.value_counts().plot(kind='bar', title='Count (target)');


# In[7]:


# from xgboost import XGBClassifier
# from sklearn.model_selection import train_test_split
# from sklearn.metrics import accuracy_score

# # Remove 'id' and 'target' columns

# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# X_test = test.drop(['MachineIdentifier'],1)

# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_rus, y_rus, test_size=0.05, random_state=1)

# model = XGBClassifier(max_depth=9)
# model.fit(x_train, y_train)
# y_pred = model.predict(x_test)
# y_pred_actual = model.predict(X_test)


# In[8]:


import lightgbm
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
# X_test = test
X_test = test.drop(['MachineIdentifier'],1)
y_train = train['HasDetections']
# s= y_train.tolist().count(1)/y_train.tolist().count(0)
X_train = train.drop(['HasDetections','MachineIdentifier'],1)
# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# sample_weight = np.random.RandomState(42).rand(y_train.shape[0])
# X_train, X_test, y_train, y_test, sw_train, sw_test = train_test_split(X_train, y_train, sample_weight, test_size=0.2, random_state=42)

x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=1,stratify=y_train)
# x_train,x_test,y_train,y_test=tts(x_over,y_over,test_size=0.20,random_state=0,stratify=y_over)
import re
x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
train_data = lightgbm.Dataset(x_train, label=y_train)
test_data = lightgbm.Dataset(x_test, label=y_test)
parameters = {
    'application': 'binary',
    'objective': 'binary',
    'metric': 'auc',
    'is_unbalance': 'true',
    'boosting': 'gbdt',
    'feature_fraction': 0.5,
    'bagging_fraction': 0.5,
    'bagging_freq': 20,
    'learning_rate': 0.01,
    'verbose': 0
}

model = lightgbm.train(parameters,
                       train_data,
                       valid_sets=test_data,
                       num_boost_round=50000,
                       early_stopping_rounds=100)

y_pred_actual = model.predict(X_test)


# In[31]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# import re
# x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier(max_depth=8)
# model_.fit(x_train, y_train)
# y_pred_ = model_.predict(x_test)
# # y_pred_actual = model.predict(X_test)


# In[41]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_sm, y_sm, test_size=0.1, random_state=1)
# import re
# x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier()
# model_.fit(x_train, y_train)
# y_pred_ = model_.predict(x_test)
# y_pred_actual = model_.predict_proba(X_test)
# print(y_pred_actual)


# In[44]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# # x_train, x_test, y_train, y_test = train_test_split(X_sm, y_sm, test_size=0.1, random_state=1)
# import re
# X_sm = X_sm.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# # x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier()
# model_.fit(X_sm, y_sm)
# y_pred_actual = model_.predict_proba(X_test)
# print(y_pred_actual)


# In[39]:


# from sklearn.linear_model import LogisticRegression
# from sklearn.model_selection import train_test_split
# from sklearn.metrics import accuracy_score

# # Remove 'id' and 'target' columns

# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# X_test = test.drop(['MachineIdentifier'],1)

# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.05, random_state=1)

# model_lr = LogisticRegression()
# model_lr.fit(x_train, y_train)
# y_pred_lr = model_lr.predict(x_test)
# y_pred_actual_lr = model_lr.predict(X_test)


# In[ ]:





# In[11]:


# from sklearn.linear_model import LogisticRegression
# from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score, RepeatedStratifiedKFold, StratifiedKFold
# from sklearn.metrics import accuracy_score, confusion_matrix,roc_curve, roc_auc_score, precision_score, recall_score, precision_recall_curve
# from sklearn.metrics import f1_score
# from sklearn import metrics
# print(f'Accuracy Score: {accuracy_score(y_test,y_pred)}')
# print(f'Confusion Matrix: \n{confusion_matrix(y_test, y_pred)}')
# print(f'Area Under Curve: {roc_auc_score(y_test, y_pred)}')
# print(f'Recall score: {recall_score(y_test,y_pred)}')
# print("Train Set Accuracy is ==> ",metrics.accuracy_score(y_train,model.predict(x_train)))
# print("Test Set Accuracy is ==> ",metrics.accuracy_score(y_test,y_pred))
# print("Classification Report on Hold Out Dataset==>\n\n",metrics.classification_report(y_test,y_pred))


# In[ ]:





# In[ ]:





# In[45]:


# # y_pred_actual = model_.predict(X_test)
# final = []
# for i in range(len(y_pred_actual)):
#     final.append(y_pred_actual[i][1])
# print(final[len(y_pred_actual)-1])
# print(len(final))


# In[ ]:


print(y_pred_actual)


# In[51]:





# In[ ]:





# In[ ]:





# In[10]:


output = pd.DataFrame(columns=['MachineIdentifier','HasDetections'])
output['MachineIdentifier']= id_
output['HasDetections'] = y_pred_actual
output.to_csv('final_output.csv',index=False)


# In[38]:


output


# In[ ]:





b5b6bc845fa1ca67ed67753d84c796919cdfc15d
<mask token>


def handle_non_numerical_data(df, columns):
    for column in columns:
        text_digit_vals = {}

        def convert_to_int(val):
            return text_digit_vals[val]
        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1
            df[column] = list(map(convert_to_int, df[column]))
    return df


<mask token>

<mask token>
train.drop(remove_cols, axis=1, inplace=True)
<mask token>
test.drop(remove_cols, axis=1, inplace=True)
<mask token>
for col in train.columns:
    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 
        100 / train.shape[0], train[col].value_counts(normalize=True,
        dropna=False).values[0] * 100, train[col].dtype))
<mask token>
stats_df.sort_values('Percentage of missing values', ascending=False)
<mask token>
for col in train.columns:
    if train[col].dtypes == object:
        unique = len(train[col].unique())
        if unique > 50:
            label.append(col)
        print('Feature {} has {} unique categories'.format(col, unique))
print(label)


def handle_non_numerical_data(df, columns):
    for column in columns:
        text_digit_vals = {}

        def convert_to_int(val):
            return text_digit_vals[val]
        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1
            df[column] = list(map(convert_to_int, df[column]))
    return df


<mask token>
for i in train['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
<mask token>
for i in test['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
<mask token>
for i in train['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in test['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
<mask token>
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
<mask token>
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
<mask token>
print(y_pred_actual)
<mask token>
output.to_csv('final_output.csv', index=False)
output

<mask token>
train = pd.read_csv(
    'C:\\Users\\Mrinal M\\Desktop\\ML_assignment\\Malware_train.csv')
test = pd.read_csv(
    'C:\\Users\\Mrinal M\\Desktop\\ML_assignment\\Malware_test.csv')
remove_cols = ['Census_InternalBatteryType', 'DefaultBrowsersIdentifier',
    'PuaMode', 'Census_ProcessorClass', 'Census_IsWIMBootEnabled', 'IsBeta',
    'Census_IsFlightsDisabled', 'Census_IsFlightingInternal',
    'AutoSampleOptIn', 'Census_ThresholdOptIn', 'SMode',
    'Census_IsPortableOperatingSystem', 'Census_DeviceFamily',
    'UacLuaenable', 'Census_IsVirtualDevice', 'Platform',
    'Census_OSSkuName', 'Census_OSInstallLanguageIdentifier', 'Processor']
train.drop(remove_cols, axis=1, inplace=True)
id_ = test['MachineIdentifier']
test.drop(remove_cols, axis=1, inplace=True)
stats = []
for col in train.columns:
    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 
        100 / train.shape[0], train[col].value_counts(normalize=True,
        dropna=False).values[0] * 100, train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values',
    'Percentage of missing values',
    'Percentage of values in the biggest category', 'type'])
stats_df.sort_values('Percentage of missing values', ascending=False)
label = []
for col in train.columns:
    if train[col].dtypes == object:
        unique = len(train[col].unique())
        if unique > 50:
            label.append(col)
        print('Feature {} has {} unique categories'.format(col, unique))
print(label)


def handle_non_numerical_data(df, columns):
    for column in columns:
        text_digit_vals = {}

        def convert_to_int(val):
            return text_digit_vals[val]
        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1
            df[column] = list(map(convert_to_int, df[column]))
    return df


train = handle_non_numerical_data(train, label)
test = handle_non_numerical_data(test, label)
l = ['mse', 'windowsintune', 'mseprerelease']
train['ProductName'] = [(1 if i in l else 0) for i in train['ProductName']]
test['ProductName'] = [(1 if i in l else 0) for i in test['ProductName']]
dummies = pd.get_dummies(train['ProductName'], prefix='ProductName',
    dummy_na=False)
train = train.drop('ProductName', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['ProductName'], prefix='ProductName',
    dummy_na=False)
test = test.drop('ProductName', 1)
test = pd.concat([test, dummies], axis=1)
l = ['6.1.1.0', '6.3.0.0', '6.1.0.0', '10.0.1.0', '6.3.1.0', '10.0.32.72',
    '10.0.80.0', '10.0.0.2']
train['OsVer'] = [(1 if i in l else 0) for i in train['OsVer']]
test['OsVer'] = [(1 if i in l else 0) for i in test['OsVer']]
dummies = pd.get_dummies(train['OsVer'], prefix='OsVer', dummy_na=False)
train = train.drop('OsVer', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['OsVer'], prefix='OsVer', dummy_na=False)
test = test.drop('OsVer', 1)
test = pd.concat([test, dummies], axis=1)
dummies = pd.get_dummies(train['OsPlatformSubRelease'], prefix=
    'OsPlatformSubRelease', dummy_na=False)
train = train.drop('OsPlatformSubRelease', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['OsPlatformSubRelease'], prefix=
    'OsPlatformSubRelease', dummy_na=False)
test = test.drop('OsPlatformSubRelease', 1)
test = pd.concat([test, dummies], axis=1)
home = ['Home']
pro = ['Pro']
other = ['Education', 'Enterprise LTSB', 'Enterprise', 'Invalid', 'Cloud',
    'Server']
SkuEdition = []
for i in train['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
train['SkuEdition'] = SkuEdition
SkuEdition = []
for i in test['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
test['SkuEdition'] = SkuEdition
dummies = pd.get_dummies(train['SkuEdition'], prefix='SkuEdition', dummy_na
    =False)
train = train.drop('SkuEdition', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['SkuEdition'], prefix='SkuEdition', dummy_na=
    False)
test = test.drop('SkuEdition', 1)
test = pd.concat([test, dummies], axis=1)
home = ['RequireAdmin']
pro = ['ExistsNotSet']
Ss = []
for i in train['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train['SmartScreen'] = Ss
Ss = []
for i in test['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test['SmartScreen'] = Ss
dummies = pd.get_dummies(train['SmartScreen'], prefix='SmartScreen',
    dummy_na=False)
train = train.drop('SmartScreen', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['SmartScreen'], prefix='SmartScreen',
    dummy_na=False)
test = test.drop('SmartScreen', 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_MDC2FormFactor'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_PrimaryDiskTypeName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_PowerPlatformRoleName'
home = ['Desktop']
pro = ['Mobile']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
col = 'Census_PowerPlatformRoleName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSArchitecture'
train[col] = [(1 if i in l else 0) for i in train[col]]
test[col] = [(1 if i in l else 0) for i in test[col]]
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_FlightRing'
home = ['Retail']
pro = ['NOT_SET']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_ActivationChannel'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_GenuineStateName'
home = ['IS_GENUINE']
pro = ['INVALID_LICENSE']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSWUAutoUpdateOptionsName'
home = ['FullAuto']
pro = ['Notify']
blah = ['AutoInstallAndRebootAtMaintenanceTime']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSInstallTypeName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSEdition'
home = ['Professional']
pro = ['CoreSingleLanguage']
blah = ['Core']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSBranch'
home = ['rs4_release']
pro = ['rs3_release']
blah = ['rs2_release']
haha = ['rs1_release']
hehe = ['rs3_release_svc_escrow']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_ChassisTypeName'
home = ['Notebook']
pro = ['Desktop']
blah = ['Laptop']
haha = ['Portable']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
<mask token>
imp = SimpleImputer(missing_values=np.NaN, strategy='median')
train = pd.DataFrame(imp.fit_transform(train), columns=train.columns)
test = pd.DataFrame(imp.fit_transform(test), columns=test.columns)
<mask token>
X_test = test.drop(['MachineIdentifier'], 1)
y_train = train['HasDetections']
X_train = train.drop(['HasDetections', 'MachineIdentifier'], 1)
x_train, x_test, y_train, y_test = train_test_split(X_train, y_train,
    test_size=0.1, random_state=1, stratify=y_train)
<mask token>
x_train = x_train.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '', x))
x_test = x_test.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '', x))
train_data = lightgbm.Dataset(x_train, label=y_train)
test_data = lightgbm.Dataset(x_test, label=y_test)
parameters = {'application': 'binary', 'objective': 'binary', 'metric':
    'auc', 'is_unbalance': 'true', 'boosting': 'gbdt', 'feature_fraction': 
    0.5, 'bagging_fraction': 0.5, 'bagging_freq': 20, 'learning_rate': 0.01,
    'verbose': 0}
model = lightgbm.train(parameters, train_data, valid_sets=test_data,
    num_boost_round=50000, early_stopping_rounds=100)
y_pred_actual = model.predict(X_test)
print(y_pred_actual)
output = pd.DataFrame(columns=['MachineIdentifier', 'HasDetections'])
output['MachineIdentifier'] = id_
output['HasDetections'] = y_pred_actual
output.to_csv('final_output.csv', index=False)
output

import pandas as pd
import numpy as np
train = pd.read_csv(
    'C:\\Users\\Mrinal M\\Desktop\\ML_assignment\\Malware_train.csv')
test = pd.read_csv(
    'C:\\Users\\Mrinal M\\Desktop\\ML_assignment\\Malware_test.csv')
remove_cols = ['Census_InternalBatteryType', 'DefaultBrowsersIdentifier',
    'PuaMode', 'Census_ProcessorClass', 'Census_IsWIMBootEnabled', 'IsBeta',
    'Census_IsFlightsDisabled', 'Census_IsFlightingInternal',
    'AutoSampleOptIn', 'Census_ThresholdOptIn', 'SMode',
    'Census_IsPortableOperatingSystem', 'Census_DeviceFamily',
    'UacLuaenable', 'Census_IsVirtualDevice', 'Platform',
    'Census_OSSkuName', 'Census_OSInstallLanguageIdentifier', 'Processor']
train.drop(remove_cols, axis=1, inplace=True)
id_ = test['MachineIdentifier']
test.drop(remove_cols, axis=1, inplace=True)
stats = []
for col in train.columns:
    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 
        100 / train.shape[0], train[col].value_counts(normalize=True,
        dropna=False).values[0] * 100, train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values',
    'Percentage of missing values',
    'Percentage of values in the biggest category', 'type'])
stats_df.sort_values('Percentage of missing values', ascending=False)
label = []
for col in train.columns:
    if train[col].dtypes == object:
        unique = len(train[col].unique())
        if unique > 50:
            label.append(col)
        print('Feature {} has {} unique categories'.format(col, unique))
print(label)


def handle_non_numerical_data(df, columns):
    for column in columns:
        text_digit_vals = {}

        def convert_to_int(val):
            return text_digit_vals[val]
        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1
            df[column] = list(map(convert_to_int, df[column]))
    return df


train = handle_non_numerical_data(train, label)
test = handle_non_numerical_data(test, label)
l = ['mse', 'windowsintune', 'mseprerelease']
train['ProductName'] = [(1 if i in l else 0) for i in train['ProductName']]
test['ProductName'] = [(1 if i in l else 0) for i in test['ProductName']]
dummies = pd.get_dummies(train['ProductName'], prefix='ProductName',
    dummy_na=False)
train = train.drop('ProductName', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['ProductName'], prefix='ProductName',
    dummy_na=False)
test = test.drop('ProductName', 1)
test = pd.concat([test, dummies], axis=1)
l = ['6.1.1.0', '6.3.0.0', '6.1.0.0', '10.0.1.0', '6.3.1.0', '10.0.32.72',
    '10.0.80.0', '10.0.0.2']
train['OsVer'] = [(1 if i in l else 0) for i in train['OsVer']]
test['OsVer'] = [(1 if i in l else 0) for i in test['OsVer']]
dummies = pd.get_dummies(train['OsVer'], prefix='OsVer', dummy_na=False)
train = train.drop('OsVer', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['OsVer'], prefix='OsVer', dummy_na=False)
test = test.drop('OsVer', 1)
test = pd.concat([test, dummies], axis=1)
dummies = pd.get_dummies(train['OsPlatformSubRelease'], prefix=
    'OsPlatformSubRelease', dummy_na=False)
train = train.drop('OsPlatformSubRelease', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['OsPlatformSubRelease'], prefix=
    'OsPlatformSubRelease', dummy_na=False)
test = test.drop('OsPlatformSubRelease', 1)
test = pd.concat([test, dummies], axis=1)
home = ['Home']
pro = ['Pro']
other = ['Education', 'Enterprise LTSB', 'Enterprise', 'Invalid', 'Cloud',
    'Server']
SkuEdition = []
for i in train['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
train['SkuEdition'] = SkuEdition
SkuEdition = []
for i in test['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
test['SkuEdition'] = SkuEdition
dummies = pd.get_dummies(train['SkuEdition'], prefix='SkuEdition', dummy_na
    =False)
train = train.drop('SkuEdition', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['SkuEdition'], prefix='SkuEdition', dummy_na=
    False)
test = test.drop('SkuEdition', 1)
test = pd.concat([test, dummies], axis=1)
home = ['RequireAdmin']
pro = ['ExistsNotSet']
Ss = []
for i in train['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train['SmartScreen'] = Ss
Ss = []
for i in test['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test['SmartScreen'] = Ss
dummies = pd.get_dummies(train['SmartScreen'], prefix='SmartScreen',
    dummy_na=False)
train = train.drop('SmartScreen', 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test['SmartScreen'], prefix='SmartScreen',
    dummy_na=False)
test = test.drop('SmartScreen', 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_MDC2FormFactor'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_PrimaryDiskTypeName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_PowerPlatformRoleName'
home = ['Desktop']
pro = ['Mobile']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
col = 'Census_PowerPlatformRoleName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSArchitecture'
train[col] = [(1 if i in l else 0) for i in train[col]]
test[col] = [(1 if i in l else 0) for i in test[col]]
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_FlightRing'
home = ['Retail']
pro = ['NOT_SET']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_ActivationChannel'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_GenuineStateName'
home = ['IS_GENUINE']
pro = ['INVALID_LICENSE']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSWUAutoUpdateOptionsName'
home = ['FullAuto']
pro = ['Notify']
blah = ['AutoInstallAndRebootAtMaintenanceTime']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSInstallTypeName'
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSEdition'
home = ['Professional']
pro = ['CoreSingleLanguage']
blah = ['Core']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_OSBranch'
home = ['rs4_release']
pro = ['rs3_release']
blah = ['rs2_release']
haha = ['rs1_release']
hehe = ['rs3_release_svc_escrow']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
col = 'Census_ChassisTypeName'
home = ['Notebook']
pro = ['Desktop']
blah = ['Laptop']
haha = ['Portable']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col, dummy_na=False)
train = train.drop(col, 1)
train = pd.concat([train, dummies], axis=1)
dummies = pd.get_dummies(test[col], prefix=col, dummy_na=False)
test = test.drop(col, 1)
test = pd.concat([test, dummies], axis=1)
from sklearn.impute import SimpleImputer
imp = SimpleImputer(missing_values=np.NaN, strategy='median')
train = pd.DataFrame(imp.fit_transform(train), columns=train.columns)
test = pd.DataFrame(imp.fit_transform(test), columns=test.columns)
import lightgbm
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
X_test = test.drop(['MachineIdentifier'], 1)
y_train = train['HasDetections']
X_train = train.drop(['HasDetections', 'MachineIdentifier'], 1)
x_train, x_test, y_train, y_test = train_test_split(X_train, y_train,
    test_size=0.1, random_state=1, stratify=y_train)
import re
x_train = x_train.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '', x))
x_test = x_test.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '', x))
train_data = lightgbm.Dataset(x_train, label=y_train)
test_data = lightgbm.Dataset(x_test, label=y_test)
parameters = {'application': 'binary', 'objective': 'binary', 'metric':
    'auc', 'is_unbalance': 'true', 'boosting': 'gbdt', 'feature_fraction': 
    0.5, 'bagging_fraction': 0.5, 'bagging_freq': 20, 'learning_rate': 0.01,
    'verbose': 0}
model = lightgbm.train(parameters, train_data, valid_sets=test_data,
    num_boost_round=50000, early_stopping_rounds=100)
y_pred_actual = model.predict(X_test)
print(y_pred_actual)
output = pd.DataFrame(columns=['MachineIdentifier', 'HasDetections'])
output['MachineIdentifier'] = id_
output['HasDetections'] = y_pred_actual
output.to_csv('final_output.csv', index=False)
output

#!/usr/bin/env python
# coding: utf-8

# In[2]:


import pandas as pd
import numpy as np


# In[3]:


train = pd.read_csv(r"C:\Users\Mrinal M\Desktop\ML_assignment\Malware_train.csv")
test = pd.read_csv(r"C:\Users\Mrinal M\Desktop\ML_assignment\Malware_test.csv")
remove_cols = ['Census_InternalBatteryType','DefaultBrowsersIdentifier','PuaMode', 'Census_ProcessorClass', 'Census_IsWIMBootEnabled', 'IsBeta', 'Census_IsFlightsDisabled', 'Census_IsFlightingInternal', 'AutoSampleOptIn', 'Census_ThresholdOptIn', 'SMode', 'Census_IsPortableOperatingSystem', 'Census_DeviceFamily', 'UacLuaenable', 'Census_IsVirtualDevice', 'Platform', 'Census_OSSkuName', 'Census_OSInstallLanguageIdentifier', 'Processor']
# The columns in remove_Cols are columns with 90percent nan values and heavy biased columns.
train.drop(remove_cols, axis=1, inplace=True)
id_ = test['MachineIdentifier']
test.drop(remove_cols, axis=1, inplace=True)


# In[4]:


stats = []
for col in train.columns:
    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 100 / train.shape[0], train[col].value_counts(normalize=True, dropna=False).values[0] * 100, train[col].dtype))
    
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])
stats_df.sort_values('Percentage of missing values', ascending=False)


# In[5]:


label = []
for col in train.columns:
    if train[col].dtypes == object:
        unique = len(train[col].unique())
        if unique>50:
            label.append(col)
        print("Feature {} has {} unique categories".format(col,unique))
print(label)
def handle_non_numerical_data(df,columns):
#     columns = df.columns.values
    for column in columns:
        text_digit_vals = {}
        def convert_to_int(val):
            return text_digit_vals[val]

        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x+=1

            df[column] = list(map(convert_to_int, df[column]))

    return df
# train = handle_non_numerical_data(train)
# test = handle_non_numerical_data(test)
train = handle_non_numerical_data(train,label)
test = handle_non_numerical_data(test,label)
# # label = ['MachineIdentifier','AvSigVersion','AVProductStatesIdentifier','CityIdentifier','Census_OEMNameIdentifier','Census_OEMModelIdentifier','Census_ProcessorModelIdentifier','Census_PrimaryDiskTotalCapacity','Census_InternalBatteryNumberOfCharges',]


# In[6]:


l = ['mse' ,'windowsintune', 'mseprerelease']
train['ProductName'] = [1 if i in l else 0 for i in train['ProductName']]
test['ProductName'] = [1 if i in l else 0 for i in test['ProductName']]
dummies = pd.get_dummies(train['ProductName'], prefix='ProductName',dummy_na=False)
train = train.drop('ProductName',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['ProductName'], prefix='ProductName',dummy_na=False)
test = test.drop('ProductName',1)
test = pd.concat([test,dummies],axis=1)
l = ['6.1.1.0' ,'6.3.0.0', '6.1.0.0', '10.0.1.0' ,'6.3.1.0','10.0.32.72', '10.0.80.0', '10.0.0.2']
train['OsVer'] = [1 if i in l else 0 for i in train['OsVer']]
test['OsVer'] = [1 if i in l else 0 for i in test['OsVer']]
dummies = pd.get_dummies(train['OsVer'], prefix='OsVer',dummy_na=False)
train = train.drop('OsVer',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['OsVer'], prefix='OsVer',dummy_na=False)
test = test.drop('OsVer',1)
test = pd.concat([test,dummies],axis=1)
dummies = pd.get_dummies(train['OsPlatformSubRelease'], prefix='OsPlatformSubRelease',dummy_na=False)
train = train.drop('OsPlatformSubRelease',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['OsPlatformSubRelease'], prefix='OsPlatformSubRelease',dummy_na=False)
test = test.drop('OsPlatformSubRelease',1)
test = pd.concat([test,dummies],axis=1)
home = ['Home']
pro = ['Pro']
other = ['Education','Enterprise LTSB','Enterprise','Invalid','Cloud','Server']
SkuEdition = []
for i in train['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
train['SkuEdition'] = SkuEdition
SkuEdition = []
for i in test['SkuEdition']:
    if i in home:
        SkuEdition.append(0)
    elif i in pro:
        SkuEdition.append(1)
    elif i in other:
        SkuEdition.append(2)
test['SkuEdition'] = SkuEdition
dummies = pd.get_dummies(train['SkuEdition'], prefix='SkuEdition',dummy_na=False)
train = train.drop('SkuEdition',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['SkuEdition'], prefix='SkuEdition',dummy_na=False)
test = test.drop('SkuEdition',1)
test = pd.concat([test,dummies],axis=1)
home = ['RequireAdmin']
pro = ['ExistsNotSet']
Ss = []
for i in train['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train['SmartScreen'] = Ss
Ss = []
for i in test['SmartScreen']:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test['SmartScreen'] = Ss
dummies = pd.get_dummies(train['SmartScreen'], prefix='SmartScreen',dummy_na=False)
train = train.drop('SmartScreen',1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test['SmartScreen'], prefix='SmartScreen',dummy_na=False)
test = test.drop('SmartScreen',1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_MDC2FormFactor'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_PrimaryDiskTypeName'  
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_PowerPlatformRoleName'
home = ['Desktop']
pro = ['Mobile']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
col = 'Census_PowerPlatformRoleName'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSArchitecture'
train[col] = [1 if i in l else 0 for i in train[col]]
test[col] = [1 if i in l else 0 for i in test[col]]
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_FlightRing'
home = ['Retail']
pro = ['NOT_SET']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_ActivationChannel'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_GenuineStateName'
home = ['IS_GENUINE']
pro = ['INVALID_LICENSE']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    else:
        Ss.append(2)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)

col = 'Census_OSWUAutoUpdateOptionsName'
home = ['FullAuto']
pro = ['Notify']
blah = ['AutoInstallAndRebootAtMaintenanceTime']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else: Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSInstallTypeName'
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSEdition'
home = ['Professional']
pro = ['CoreSingleLanguage']
blah = ['Core']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else: Ss.append(3)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    else:
        Ss.append(3)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_OSBranch'
home = ['rs4_release']
pro = ['rs3_release']
blah = ['rs2_release']
haha = ['rs1_release']
hehe = ['rs3_release_svc_escrow']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    elif i in hehe:
        Ss.append(4)
    else:
        Ss.append(5)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)
col = 'Census_ChassisTypeName'
home = ['Notebook']
pro = ['Desktop']
blah = ['Laptop']
haha = ['Portable']
Ss = []
for i in train[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
train[col] = Ss
Ss = []
for i in test[col]:
    if i in home:
        Ss.append(0)
    elif i in pro:
        Ss.append(1)
    elif i in blah:
        Ss.append(2)
    elif i in haha:
        Ss.append(3)
    else:
        Ss.append(4)
test[col] = Ss
dummies = pd.get_dummies(train[col], prefix=col,dummy_na=False)
train = train.drop(col,1)
train = pd.concat([train,dummies],axis=1)
dummies = pd.get_dummies(test[col], prefix=col,dummy_na=False)
test = test.drop(col,1)
test = pd.concat([test,dummies],axis=1)


# In[7]:


from sklearn.impute import SimpleImputer
imp = SimpleImputer(missing_values=np.NaN, strategy = 'median')
train = pd.DataFrame(imp.fit_transform(train),columns=train.columns)
test = pd.DataFrame(imp.fit_transform(test),columns=test.columns)


# In[6]:


# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# from imblearn.under_sampling import RandomUnderSampler
# rus = RandomUnderSampler()
# X_rus, y_rus= rus.fit_sample(X_train, y_train)


# In[28]:


# from imblearn.over_sampling import SMOTE
# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# smote = SMOTE(sampling_strategy='minority')
# X_sm, y_sm = smote.fit_sample(X_train, y_train)


# In[15]:


# # Class count
# count_class_0, count_class_1 = train.HasDetections.value_counts()

# # Divide by class
# df_class_0 = train[train['HasDetections'] == 0]
# df_class_1 = train[train['HasDetections'] == 1]


# In[16]:


# df_class_0_under = df_class_0.sample(count_class_1)
# train = pd.concat([df_class_0_under, df_class_1], axis=0)

# print('Random under-sampling:')
# print(train.HasDetections.value_counts())

# train.HasDetections.value_counts().plot(kind='bar', title='Count (target)');


# In[7]:


# from xgboost import XGBClassifier
# from sklearn.model_selection import train_test_split
# from sklearn.metrics import accuracy_score

# # Remove 'id' and 'target' columns

# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# X_test = test.drop(['MachineIdentifier'],1)

# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_rus, y_rus, test_size=0.05, random_state=1)

# model = XGBClassifier(max_depth=9)
# model.fit(x_train, y_train)
# y_pred = model.predict(x_test)
# y_pred_actual = model.predict(X_test)


# In[8]:


import lightgbm
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
# X_test = test
X_test = test.drop(['MachineIdentifier'],1)
y_train = train['HasDetections']
# s= y_train.tolist().count(1)/y_train.tolist().count(0)
X_train = train.drop(['HasDetections','MachineIdentifier'],1)
# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# sample_weight = np.random.RandomState(42).rand(y_train.shape[0])
# X_train, X_test, y_train, y_test, sw_train, sw_test = train_test_split(X_train, y_train, sample_weight, test_size=0.2, random_state=42)

x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=1,stratify=y_train)
# x_train,x_test,y_train,y_test=tts(x_over,y_over,test_size=0.20,random_state=0,stratify=y_over)
import re
x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
train_data = lightgbm.Dataset(x_train, label=y_train)
test_data = lightgbm.Dataset(x_test, label=y_test)
parameters = {
    'application': 'binary',
    'objective': 'binary',
    'metric': 'auc',
    'is_unbalance': 'true',
    'boosting': 'gbdt',
    'feature_fraction': 0.5,
    'bagging_fraction': 0.5,
    'bagging_freq': 20,
    'learning_rate': 0.01,
    'verbose': 0
}

model = lightgbm.train(parameters,
                       train_data,
                       valid_sets=test_data,
                       num_boost_round=50000,
                       early_stopping_rounds=100)

y_pred_actual = model.predict(X_test)


# In[31]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# import re
# x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier(max_depth=8)
# model_.fit(x_train, y_train)
# y_pred_ = model_.predict(x_test)
# # y_pred_actual = model.predict(X_test)


# In[41]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# x_train, x_test, y_train, y_test = train_test_split(X_sm, y_sm, test_size=0.1, random_state=1)
# import re
# x_train = x_train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier()
# model_.fit(x_train, y_train)
# y_pred_ = model_.predict(x_test)
# y_pred_actual = model_.predict_proba(X_test)
# print(y_pred_actual)


# In[44]:


# from sklearn import *
# from lightgbm import LGBMClassifier
# X_test = test.drop(['MachineIdentifier'],1)
# # y_train = train['HasDetections']
# # X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# # x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=1)
# # x_train, x_test, y_train, y_test = train_test_split(X_sm, y_sm, test_size=0.1, random_state=1)
# import re
# X_sm = X_sm.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# # x_test = x_test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
# model_ = LGBMClassifier()
# model_.fit(X_sm, y_sm)
# y_pred_actual = model_.predict_proba(X_test)
# print(y_pred_actual)


# In[39]:


# from sklearn.linear_model import LogisticRegression
# from sklearn.model_selection import train_test_split
# from sklearn.metrics import accuracy_score

# # Remove 'id' and 'target' columns

# y_train = train['HasDetections']
# X_train = train.drop(['MachineIdentifier','HasDetections'],1)
# X_test = test.drop(['MachineIdentifier'],1)

# x_train, x_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.05, random_state=1)

# model_lr = LogisticRegression()
# model_lr.fit(x_train, y_train)
# y_pred_lr = model_lr.predict(x_test)
# y_pred_actual_lr = model_lr.predict(X_test)


# In[ ]:





# In[11]:


# from sklearn.linear_model import LogisticRegression
# from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score, RepeatedStratifiedKFold, StratifiedKFold
# from sklearn.metrics import accuracy_score, confusion_matrix,roc_curve, roc_auc_score, precision_score, recall_score, precision_recall_curve
# from sklearn.metrics import f1_score
# from sklearn import metrics
# print(f'Accuracy Score: {accuracy_score(y_test,y_pred)}')
# print(f'Confusion Matrix: \n{confusion_matrix(y_test, y_pred)}')
# print(f'Area Under Curve: {roc_auc_score(y_test, y_pred)}')
# print(f'Recall score: {recall_score(y_test,y_pred)}')
# print("Train Set Accuracy is ==> ",metrics.accuracy_score(y_train,model.predict(x_train)))
# print("Test Set Accuracy is ==> ",metrics.accuracy_score(y_test,y_pred))
# print("Classification Report on Hold Out Dataset==>\n\n",metrics.classification_report(y_test,y_pred))


# In[ ]:





# In[ ]:





# In[45]:


# # y_pred_actual = model_.predict(X_test)
# final = []
# for i in range(len(y_pred_actual)):
#     final.append(y_pred_actual[i][1])
# print(final[len(y_pred_actual)-1])
# print(len(final))


# In[ ]:


print(y_pred_actual)


# In[51]:





# In[ ]:





# In[ ]:





# In[10]:


output = pd.DataFrame(columns=['MachineIdentifier','HasDetections'])
output['MachineIdentifier']= id_
output['HasDetections'] = y_pred_actual
output.to_csv('final_output.csv',index=False)


# In[38]:


output


# In[ ]:




