import numpy as np
import pandas as pd

from sklearn.model_selection import KFold
from sklearn.metrics import log_loss, mean_squared_error

from .model_selection import PurgedKFold, cv_score, evaluate


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring='neg_log_loss',
                    n_splits=3, t1=None, cv_gen=None, pct_embargo=0, purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y,
                          sample_weight=sample_weight,
                          scoring=scoring,
                          cv_gen=cv_gen,
                          n_splits=n_splits,
                          t1=t1,
                          pct_embargo=pct_embargo,
                          purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
                 n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True, num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
                        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
                        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
                        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({"mean": df0.mean(), "std": df0.std() * (df0.shape[0] ** -0.5)}, axis=1)
    imp /= imp["mean"].sum()
    return imp


def feat_imp_MDA(clf, X, y, sample_weight=None, scoring='neg_log_loss', n_splits=5, t1=None,
                 cv_gen=None, pct_embargo=0, purging=True, num_threads=1):
    """Calculate Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label        
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=pct_embargo,
                                 purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=X.columns)
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
                                   sample_weight=w_test)

        for col in X.columns:
            X_test_ = X_test.copy(deep=True)
            # Randomize certain feature to make it not effective
            np.random.shuffle(X_test_[col].values)
            scores_perm.loc[idx, col] = evaluate(clf_fit, X_test_, y_test, scoring,
                                                 sample_weight=w_test)
    # (Original score) - (premutated score)
    imprv = (-scores_perm).add(scores, axis=0)
    # Relative to maximum improvement
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1. - scores_perm
    imp = imprv / max_imprv
    return pd.concat({"mean": imp.mean(), "std": imp.std() * (imp.shape[0] ** -0.5)}, axis=1)

def group_mean_std(df0, clstrs):
    out = pd.DataFrame(columns=['mean', 'std'])
    for key, elements in clstrs.items():
        df1 = df0[elements].sum(axis=1)
        out.loc[f"C_{key}", 'mean'] = df1.mean()
        out.loc[f"C_{key}", 'std'] = df1.std() * df1.shape[0]**-.5
    return out

def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i:tree.feature_importances_ for i, tree in enumerate(fit.estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan) #because max_features=1
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, 
                           sample_weight=None,
                           scoring='neg_log_loss',
                           n_splits=5, t1=None,
                           cv_gen=None, pct_embargo=0,
                           purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=pct_embargo,
                                 purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
                                   sample_weight=w_test)

        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_, y_test,
                                                        scoring, sample_weight=w_test)
    # (Original score) - (premutated score)
    imprv = (-scores_perm).add(scores, axis=0)
    # Relative to maximum improvement
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1. - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] ** -0.5}, axis=1)
    imp.index = [f"C_{i}" for i in imp.index]
    return imp
ec1a7513331161f579977689909e5c9a6113a286
<mask token>


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring=
    'neg_log_loss', n_splits=3, t1=None, cv_gen=None, pct_embargo=0,
    purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y, sample_weight=
            sample_weight, scoring=scoring, cv_gen=cv_gen, n_splits=
            n_splits, t1=t1, pct_embargo=pct_embargo, purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({'mean': df0.mean(), 'std': df0.std() * df0.shape[0] **
        -0.5}, axis=1)
    imp /= imp['mean'].sum()
    return imp


<mask token>


def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, sample_weight=None, scoring=
    'neg_log_loss', n_splits=5, t1=None, cv_gen=None, pct_embargo=0,
    purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_,
                y_test, scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)
    imp.index = [f'C_{i}' for i in imp.index]
    return imp

<mask token>


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring=
    'neg_log_loss', n_splits=3, t1=None, cv_gen=None, pct_embargo=0,
    purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y, sample_weight=
            sample_weight, scoring=scoring, cv_gen=cv_gen, n_splits=
            n_splits, t1=t1, pct_embargo=pct_embargo, purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({'mean': df0.mean(), 'std': df0.std() * df0.shape[0] **
        -0.5}, axis=1)
    imp /= imp['mean'].sum()
    return imp


<mask token>


def group_mean_std(df0, clstrs):
    out = pd.DataFrame(columns=['mean', 'std'])
    for key, elements in clstrs.items():
        df1 = df0[elements].sum(axis=1)
        out.loc[f'C_{key}', 'mean'] = df1.mean()
        out.loc[f'C_{key}', 'std'] = df1.std() * df1.shape[0] ** -0.5
    return out


def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, sample_weight=None, scoring=
    'neg_log_loss', n_splits=5, t1=None, cv_gen=None, pct_embargo=0,
    purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_,
                y_test, scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)
    imp.index = [f'C_{i}' for i in imp.index]
    return imp

<mask token>


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring=
    'neg_log_loss', n_splits=3, t1=None, cv_gen=None, pct_embargo=0,
    purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y, sample_weight=
            sample_weight, scoring=scoring, cv_gen=cv_gen, n_splits=
            n_splits, t1=t1, pct_embargo=pct_embargo, purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({'mean': df0.mean(), 'std': df0.std() * df0.shape[0] **
        -0.5}, axis=1)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label        
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=X.columns)
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for col in X.columns:
            X_test_ = X_test.copy(deep=True)
            np.random.shuffle(X_test_[col].values)
            scores_perm.loc[idx, col] = evaluate(clf_fit, X_test_, y_test,
                scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    return pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)


def group_mean_std(df0, clstrs):
    out = pd.DataFrame(columns=['mean', 'std'])
    for key, elements in clstrs.items():
        df1 = df0[elements].sum(axis=1)
        out.loc[f'C_{key}', 'mean'] = df1.mean()
        out.loc[f'C_{key}', 'std'] = df1.std() * df1.shape[0] ** -0.5
    return out


def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, sample_weight=None, scoring=
    'neg_log_loss', n_splits=5, t1=None, cv_gen=None, pct_embargo=0,
    purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_,
                y_test, scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)
    imp.index = [f'C_{i}' for i in imp.index]
    return imp

import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss, mean_squared_error
from .model_selection import PurgedKFold, cv_score, evaluate


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring=
    'neg_log_loss', n_splits=3, t1=None, cv_gen=None, pct_embargo=0,
    purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y, sample_weight=
            sample_weight, scoring=scoring, cv_gen=cv_gen, n_splits=
            n_splits, t1=t1, pct_embargo=pct_embargo, purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({'mean': df0.mean(), 'std': df0.std() * df0.shape[0] **
        -0.5}, axis=1)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA(clf, X, y, sample_weight=None, scoring='neg_log_loss',
    n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True,
    num_threads=1):
    """Calculate Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label        
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=X.columns)
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for col in X.columns:
            X_test_ = X_test.copy(deep=True)
            np.random.shuffle(X_test_[col].values)
            scores_perm.loc[idx, col] = evaluate(clf_fit, X_test_, y_test,
                scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    return pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)


def group_mean_std(df0, clstrs):
    out = pd.DataFrame(columns=['mean', 'std'])
    for key, elements in clstrs.items():
        df1 = df0[elements].sum(axis=1)
        out.loc[f'C_{key}', 'mean'] = df1.mean()
        out.loc[f'C_{key}', 'std'] = df1.std() * df1.shape[0] ** -0.5
    return out


def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.
        estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, sample_weight=None, scoring=
    'neg_log_loss', n_splits=5, t1=None, cv_gen=None, pct_embargo=0,
    purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=
                pct_embargo, purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
            sample_weight=w_test)
        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_,
                y_test, scoring, sample_weight=w_test)
    imprv = (-scores_perm).add(scores, axis=0)
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1.0 - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] **
        -0.5}, axis=1)
    imp.index = [f'C_{i}' for i in imp.index]
    return imp

import numpy as np
import pandas as pd

from sklearn.model_selection import KFold
from sklearn.metrics import log_loss, mean_squared_error

from .model_selection import PurgedKFold, cv_score, evaluate


def mp_feat_imp_SFI(clf, X, y, feat_names, sample_weight=None, scoring='neg_log_loss',
                    n_splits=3, t1=None, cv_gen=None, pct_embargo=0, purging=True):
    imp = pd.DataFrame(columns=['mean', 'std'])
    for feat_name in feat_names:
        scores = cv_score(clf, X=X[[feat_name]], y=y,
                          sample_weight=sample_weight,
                          scoring=scoring,
                          cv_gen=cv_gen,
                          n_splits=n_splits,
                          t1=t1,
                          pct_embargo=pct_embargo,
                          purging=purging)
        imp.loc[feat_name, 'mean'] = scores.mean()
        imp.loc[feat_name, 'std'] = scores.std() * scores.shape[0] ** -0.5
    return imp


def feat_imp_SFI(clf, X, y, sample_weight=None, scoring='neg_log_loss',
                 n_splits=5, t1=None, cv_gen=None, pct_embargo=0, purging=True, num_threads=1):
    """Calculate Single Feature Importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
        
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    imp = mp_pandas_obj(mp_feat_imp_SFI, ('feat_names', X.columns),
                        num_threads, clf=clf, X=X, y=y, sample_weight=sample_weight,
                        scoring=scoring, n_splits=n_splits, t1=t1, cv_gen=cv_gen,
                        pct_embargo=pct_embargo, purging=purging)
    return imp


def feat_imp_MDI(fit, feat_names):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i: tree.feature_importances_ for i, tree in enumerate(fit.estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan)
    imp = pd.concat({"mean": df0.mean(), "std": df0.std() * (df0.shape[0] ** -0.5)}, axis=1)
    imp /= imp["mean"].sum()
    return imp


def feat_imp_MDA(clf, X, y, sample_weight=None, scoring='neg_log_loss', n_splits=5, t1=None,
                 cv_gen=None, pct_embargo=0, purging=True, num_threads=1):
    """Calculate Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label        
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=pct_embargo,
                                 purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=X.columns)
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
                                   sample_weight=w_test)

        for col in X.columns:
            X_test_ = X_test.copy(deep=True)
            # Randomize certain feature to make it not effective
            np.random.shuffle(X_test_[col].values)
            scores_perm.loc[idx, col] = evaluate(clf_fit, X_test_, y_test, scoring,
                                                 sample_weight=w_test)
    # (Original score) - (premutated score)
    imprv = (-scores_perm).add(scores, axis=0)
    # Relative to maximum improvement
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1. - scores_perm
    imp = imprv / max_imprv
    return pd.concat({"mean": imp.mean(), "std": imp.std() * (imp.shape[0] ** -0.5)}, axis=1)

def group_mean_std(df0, clstrs):
    out = pd.DataFrame(columns=['mean', 'std'])
    for key, elements in clstrs.items():
        df1 = df0[elements].sum(axis=1)
        out.loc[f"C_{key}", 'mean'] = df1.mean()
        out.loc[f"C_{key}", 'std'] = df1.std() * df1.shape[0]**-.5
    return out

def feat_imp_MDI_clustered(fit, feat_names, clstrs):
    """Compute Mean Decrease Impurity
    
    Args:
        forest (Forest Classifier instance)
        feat_names (list(str)): List of names of features
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns

    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    df0 = {i:tree.feature_importances_ for i, tree in enumerate(fit.estimators_)}
    df0 = pd.DataFrame.from_dict(df0, orient='index')
    df0.columns = feat_names
    df0 = df0.replace(0, np.nan) #because max_features=1
    imp = group_mean_std(df0, clstrs)
    imp /= imp['mean'].sum()
    return imp


def feat_imp_MDA_clustered(clf, X, y, clstrs, 
                           sample_weight=None,
                           scoring='neg_log_loss',
                           n_splits=5, t1=None,
                           cv_gen=None, pct_embargo=0,
                           purging=True, num_threads=1):
    """Calculate Clustered Mean Decrease Accuracy

    Note:
        You can use any classifier to estimate importance
    
    Args:
        clf: Classifier instance
        X: pd.DataFrame, Input feature
        y: pd.Series, Label
        clstrs: dict[list]
            Clustering labels: key is the name of cluster and value is list of belonging columns  
        sample_weight: pd.Series, optional
            If specified, apply this to testing and training
        scoring: str, default 'neg_log_loss'
            The name of scoring methods. 'f1', 'accuracy' or 'neg_log_loss'
        n_splits: int, default 3
            The number of splits for cross validation
        t1: pd.Series
            Index and value correspond to the begining and end of information. It is required for purging and embargo
        cv_gen: KFold instance
            If not specified, use PurgedKfold
        pct_embargo: float, default 0
            The percentage of applying embargo
        purging: bool, default True
            If true, apply purging method
        num_threads: int, default 1
            The number of threads for purging
    
    Returns:
        pd.DataFrame: Importance means and standard deviations
            - mean: Mean of importance
            - std: Standard deviation of importance
    """
    
    if cv_gen is None:
        if t1 is not None:
            cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=pct_embargo,
                                 purging=purging, num_threads=num_threads)
        else:
            cv_gen = KFold(n_splits=n_splits)
    index = np.arange(n_splits)
    scores = pd.Series(index=index)
    scores_perm = pd.DataFrame(index=index, columns=clstrs.keys())
    for idx, (train, test) in zip(index, cv_gen.split(X=X)):
        X_train = X.iloc[train]
        y_train = y.iloc[train]
        if sample_weight is not None:
            w_train = sample_weight.iloc[train].values
        else:
            w_train = None
        X_test = X.iloc[test]
        y_test = y.iloc[test]
        if sample_weight is not None:
            w_test = sample_weight.iloc[test].values
        else:
            w_test = None
        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train)
        scores.loc[idx] = evaluate(clf_fit, X_test, y_test, scoring,
                                   sample_weight=w_test)

        for clstr_name in clstrs.keys():
            X_test_ = X_test.copy(deep=True)
            for k in clstrs[clstr_name]:
                np.random.shuffle(X_test_[k].values)
            scores_perm.loc[idx, clstr_name] = evaluate(clf_fit, X_test_, y_test,
                                                        scoring, sample_weight=w_test)
    # (Original score) - (premutated score)
    imprv = (-scores_perm).add(scores, axis=0)
    # Relative to maximum improvement
    if scoring == 'neg_log_loss':
        max_imprv = -scores_perm
    else:
        max_imprv = 1. - scores_perm
    imp = imprv / max_imprv
    imp = pd.concat({'mean': imp.mean(), 'std': imp.std() * imp.shape[0] ** -0.5}, axis=1)
    imp.index = [f"C_{i}" for i in imp.index]
    return imp