


'''simple docstring'''






import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


__A :  List[str]    							=				{
    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
    "gpt2": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}







def     UpperCamelCase_      (   A__ :       Any ):
		'''simple docstring'''





		assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
		assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
		if args.mlm:
				assert os.path.isfile(args.token_counts )
				assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
		else:
				assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

		assert args.teacher_type == args.student_type or (
		    args.student_type == "distilbert" and args.teacher_type == "bert"
		)
		assert os.path.isfile(args.student_config )
		if args.student_pretrained_weights is not None:
				assert os.path.isfile(args.student_pretrained_weights )

		if args.freeze_token_type_embds:
				assert args.student_type in ["roberta"]

		assert args.alpha_ce >= 0.0
		assert args.alpha_mlm >= 0.0
		assert args.alpha_clm >= 0.0
		assert args.alpha_mse >= 0.0
		assert args.alpha_cos >= 0.0
		assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0







def     UpperCamelCase_      (   A__ :       Union[str, Any]							,   A__ :       Any ):
		'''simple docstring'''





		if args.student_type == "roberta":
				lowerCAmelCase_      :				Any      =       False
		elif args.student_type == "gpt2":
				lowerCAmelCase_      :				List[Any]      =       False







def     UpperCamelCase_      (   A__ :       Any							,   A__ :       Dict ):
		'''simple docstring'''





		if args.student_type == "roberta":
				lowerCAmelCase_      :				Optional[int]      =       False







def     UpperCamelCase_      (   ):
		'''simple docstring'''





		lowerCAmelCase_      :				List[str]      =       argparse.ArgumentParser(description="""Training""" )
		parser.add_argument("""--force"""							,   action="""store_true"""							,   help="""Overwrite dump_path if it already exists.""" )

		parser.add_argument(
		    """--dump_path"""							,   type=A__							,   required=A__							,   help="""The output directory (log, checkpoints, parameters, etc.)""" )
		parser.add_argument(
		    """--data_file"""							,   type=A__							,   required=A__							,   help="""The binarized file (tokenized + tokens_to_ids) and grouped by sequence."""							,   )

		parser.add_argument(
		    """--student_type"""							,   type=A__							,   choices=["""distilbert""", """roberta""", """gpt2"""]							,   required=A__							,   help="""The student type (DistilBERT, RoBERTa)."""							,   )
		parser.add_argument("""--student_config"""							,   type=A__							,   required=A__							,   help="""Path to the student configuration.""" )
		parser.add_argument(
		    """--student_pretrained_weights"""							,   default=A__							,   type=A__							,   help="""Load student initialization checkpoint.""" )

		parser.add_argument(
		    """--teacher_type"""							,   choices=["""bert""", """roberta""", """gpt2"""]							,   required=A__							,   help="""Teacher type (BERT, RoBERTa).""" )
		parser.add_argument("""--teacher_name"""							,   type=A__							,   required=A__							,   help="""The teacher model.""" )

		parser.add_argument("""--temperature"""							,   default=2.0							,   type=A__							,   help="""Temperature for the softmax temperature.""" )
		parser.add_argument(
		    """--alpha_ce"""							,   default=0.5							,   type=A__							,   help="""Linear weight for the distillation loss. Must be >=0.""" )
		parser.add_argument(
		    """--alpha_mlm"""							,   default=0.0							,   type=A__							,   help="""Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."""							,   )
		parser.add_argument("""--alpha_clm"""							,   default=0.5							,   type=A__							,   help="""Linear weight for the CLM loss. Must be >=0.""" )
		parser.add_argument("""--alpha_mse"""							,   default=0.0							,   type=A__							,   help="""Linear weight of the MSE loss. Must be >=0.""" )
		parser.add_argument(
		    """--alpha_cos"""							,   default=0.0							,   type=A__							,   help="""Linear weight of the cosine embedding loss. Must be >=0.""" )

		parser.add_argument(
		    """--mlm"""							,   action="""store_true"""							,   help="""The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM.""" )
		parser.add_argument(
		    """--mlm_mask_prop"""							,   default=0.15							,   type=A__							,   help="""Proportion of tokens for which we need to make a prediction."""							,   )
		parser.add_argument("""--word_mask"""							,   default=0.8							,   type=A__							,   help="""Proportion of tokens to mask out.""" )
		parser.add_argument("""--word_keep"""							,   default=0.1							,   type=A__							,   help="""Proportion of tokens to keep.""" )
		parser.add_argument("""--word_rand"""							,   default=0.1							,   type=A__							,   help="""Proportion of tokens to randomly replace.""" )
		parser.add_argument(
		    """--mlm_smoothing"""							,   default=0.7							,   type=A__							,   help="""Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."""							,   )
		parser.add_argument("""--token_counts"""							,   type=A__							,   help="""The token counts in the data_file for MLM.""" )

		parser.add_argument(
		    """--restrict_ce_to_mask"""							,   action="""store_true"""							,   help="""If true, compute the distillation loss only the [MLM] prediction distribution."""							,   )
		parser.add_argument(
		    """--freeze_pos_embs"""							,   action="""store_true"""							,   help="""Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."""							,   )
		parser.add_argument(
		    """--freeze_token_type_embds"""							,   action="""store_true"""							,   help="""Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."""							,   )

		parser.add_argument("""--n_epoch"""							,   type=A__							,   default=3							,   help="""Number of pass on the whole dataset.""" )
		parser.add_argument("""--batch_size"""							,   type=A__							,   default=5							,   help="""Batch size (for each process).""" )
		parser.add_argument(
		    """--group_by_size"""							,   action="""store_false"""							,   help="""If true, group sequences that have similar length into the same batch. Default is true."""							,   )

		parser.add_argument(
		    """--gradient_accumulation_steps"""							,   type=A__							,   default=50							,   help="""Gradient accumulation for larger training batches."""							,   )
		parser.add_argument("""--warmup_prop"""							,   default=0.05							,   type=A__							,   help="""Linear warmup proportion.""" )
		parser.add_argument("""--weight_decay"""							,   default=0.0							,   type=A__							,   help="""Weight decay if we apply some.""" )
		parser.add_argument("""--learning_rate"""							,   default=5E-4							,   type=A__							,   help="""The initial learning rate for Adam.""" )
		parser.add_argument("""--adam_epsilon"""							,   default=1E-6							,   type=A__							,   help="""Epsilon for Adam optimizer.""" )
		parser.add_argument("""--max_grad_norm"""							,   default=5.0							,   type=A__							,   help="""Max gradient norm.""" )
		parser.add_argument("""--initializer_range"""							,   default=0.02							,   type=A__							,   help="""Random initialization range.""" )

		parser.add_argument(
		    """--fp16"""							,   action="""store_true"""							,   help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""							,   )
		parser.add_argument(
		    """--fp16_opt_level"""							,   type=A__							,   default="""O1"""							,   help=(
		        """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
		        """See details at https://nvidia.github.io/apex/amp.html"""
		    )							,   )
		parser.add_argument("""--n_gpu"""							,   type=A__							,   default=1							,   help="""Number of GPUs in the node.""" )
		parser.add_argument("""--local_rank"""							,   type=A__							,   default=-1							,   help="""Distributed training - Local rank""" )
		parser.add_argument("""--seed"""							,   type=A__							,   default=56							,   help="""Random seed""" )

		parser.add_argument("""--log_interval"""							,   type=A__							,   default=5_00							,   help="""Tensorboard logging interval.""" )
		parser.add_argument("""--checkpoint_interval"""							,   type=A__							,   default=40_00							,   help="""Checkpoint interval.""" )
		lowerCAmelCase_      :				List[str]      =       parser.parse_args()
		sanity_checks(A__ )

		# ARGS #
		init_gpu_params(A__ )
		set_seed(A__ )
		if args.is_master:
				if os.path.exists(args.dump_path ):
						if not args.force:
								raise ValueError(
								    f'Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'
								    """ itUse `--force` if you want to overwrite it""" )
						else:
								shutil.rmtree(args.dump_path )

				if not os.path.exists(args.dump_path ):
						os.makedirs(args.dump_path )
				logger.info(f'Experiment will be dumped and logged in {args.dump_path}' )

				# SAVE PARAMS #
				logger.info(f'Param: {args}' )
				with open(os.path.join(args.dump_path							,   """parameters.json""" )							,   """w""" ) as f:
						json.dump(vars(A__ )							,   A__							,   indent=4 )
				git_log(args.dump_path )

		lowerCAmelCase_      :				Tuple      =       MODEL_CLASSES[args.student_type]
		lowerCAmelCase_      :				Dict      =       MODEL_CLASSES[args.teacher_type]

		# TOKENIZER #
		lowerCAmelCase_      :				Union[str, Any]      =       teacher_tokenizer_class.from_pretrained(args.teacher_name )
		lowerCAmelCase_      :				str      =       {}
		for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
				lowerCAmelCase_      :				Optional[Any]      =       tokenizer.all_special_tokens.index(A__ )
				lowerCAmelCase_      :				List[Any]      =       tokenizer.all_special_ids[idx]
		logger.info(f'Special tokens {special_tok_ids}' )
		lowerCAmelCase_      :				Optional[Any]      =       special_tok_ids
		lowerCAmelCase_      :				Union[str, Any]      =       tokenizer.max_model_input_sizes[args.teacher_name]

		# DATA LOADER #
		logger.info(f'Loading data from {args.data_file}' )
		with open(args.data_file							,   """rb""" ) as fp:
				lowerCAmelCase_      :				Union[str, Any]      =       pickle.load(A__ )

		if args.mlm:
				logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)' )
				with open(args.token_counts							,   """rb""" ) as fp:
						lowerCAmelCase_      :				Union[str, Any]      =       pickle.load(A__ )

				lowerCAmelCase_      :				Any      =       np.maximum(A__							,   1 ) ** -args.mlm_smoothing
				for idx in special_tok_ids.values():
						lowerCAmelCase_      :				Union[str, Any]      =       0.0  # do not predict special tokens
				lowerCAmelCase_      :				str      =       torch.from_numpy(A__ )
		else:
				lowerCAmelCase_      :				Optional[Any]      =       None

		lowerCAmelCase_      :				Union[str, Any]      =       LmSeqsDataset(params=A__							,   data=A__ )
		logger.info("""Data loader created.""" )

		# STUDENT #
		logger.info(f'Loading student config from {args.student_config}' )
		lowerCAmelCase_      :				str      =       student_config_class.from_pretrained(args.student_config )
		lowerCAmelCase_      :				Dict      =       True

		if args.student_pretrained_weights is not None:
				logger.info(f'Loading pretrained weights from {args.student_pretrained_weights}' )
				lowerCAmelCase_      :				List[Any]      =       student_model_class.from_pretrained(args.student_pretrained_weights							,   config=A__ )
		else:
				lowerCAmelCase_      :				Dict      =       student_model_class(A__ )

		if args.n_gpu > 0:
				student.to(f'cuda:{args.local_rank}' )
		logger.info("""Student loaded.""" )

		# TEACHER #
		lowerCAmelCase_      :				List[Any]      =       teacher_model_class.from_pretrained(args.teacher_name							,   output_hidden_states=A__ )
		if args.n_gpu > 0:
				teacher.to(f'cuda:{args.local_rank}' )
		logger.info(f'Teacher loaded from {args.teacher_name}.' )

		# FREEZING #
		if args.freeze_pos_embs:
				freeze_pos_embeddings(A__							,   A__ )
		if args.freeze_token_type_embds:
				freeze_token_type_embeddings(A__							,   A__ )

		# SANITY CHECKS #
		assert student.config.vocab_size == teacher.config.vocab_size
		assert student.config.hidden_size == teacher.config.hidden_size
		assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
		if args.mlm:
				assert token_probs.size(0 ) == stu_architecture_config.vocab_size

		# DISTILLER #
		torch.cuda.empty_cache()
		lowerCAmelCase_      :				Optional[int]      =       Distiller(
		    params=A__							,   dataset=A__							,   token_probs=A__							,   student=A__							,   teacher=A__ )
		distiller.train()
		logger.info("""Let's go get some drinks.""" )


if __name__ == "__main__":
		main()







'''simple docstring'''






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


__A :  str    							=				"Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"







def     UpperCamelCase_      (   ):
  '''simple docstring'''





  lowerCAmelCase_      :				List[Any]      =       _ask_options(
      """In which compute environment are you running?"""							,   ["""This machine""", """AWS (Amazon SageMaker)"""]							,   _convert_compute_environment							,   )
  if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
    lowerCAmelCase_      :				str      =       get_sagemaker_input()
  else:
    lowerCAmelCase_      :				Optional[int]      =       get_cluster_input()
  return config







def     UpperCamelCase_      (   A__ :       Optional[Any]=None ):
  '''simple docstring'''





  if subparsers is not None:
    lowerCAmelCase_      :				List[str]      =       subparsers.add_parser("""config"""							,   description=A__ )
  else:
    lowerCAmelCase_      :				Optional[int]      =       argparse.ArgumentParser("""Accelerate config command"""							,   description=A__ )

  parser.add_argument(
      """--config_file"""							,   default=A__							,   help=(
          """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
          """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
          """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
          """with 'huggingface'."""
      )							,   )

  if subparsers is not None:
    parser.set_defaults(func=A__ )
  return parser







def     UpperCamelCase_      (   A__ :       Any ):
  '''simple docstring'''





  lowerCAmelCase_      :				Dict      =       get_user_input()
  if args.config_file is not None:
    lowerCAmelCase_      :				List[str]      =       args.config_file
  else:
    if not os.path.isdir(A__ ):
      os.makedirs(A__ )
    lowerCAmelCase_      :				List[Any]      =       default_yaml_config_file

  if config_file.endswith(""".json""" ):
    config.to_json_file(A__ )
  else:
    config.to_yaml_file(A__ )
  print(f'accelerate configuration saved at {config_file}' )







def     UpperCamelCase_      (   ):
  '''simple docstring'''





  lowerCAmelCase_      :				str      =       config_command_parser()
  lowerCAmelCase_      :				Tuple      =       parser.parse_args()
  config_command(A__ )


if __name__ == "__main__":
  main()



