

"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a							=							logging.get_logger(__name__)

_a							=							{
    'transfo-xl-wt103': 'https://huggingface.co/transfo-xl-wt103/resolve/main/config.json',
}




class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 __UpperCAmelCase     : Dict       		=     "transfo-xl"
 __UpperCAmelCase     : Union[str, Any]       		=     ["mems"]
 __UpperCAmelCase     : Tuple       		=     {
     "n_token": "vocab_size",
     "hidden_size": "d_model",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }


 def __init__(      self							:      int, UpperCAmelCase__							:      Optional[Any]=2_6_7_7_3_5, UpperCAmelCase__							:      int=[2_0_0_0_0, 4_0_0_0_0, 2_0_0_0_0_0], UpperCAmelCase__							:      Optional[Any]=1_0_2_4, UpperCAmelCase__							:      str=1_0_2_4, UpperCAmelCase__							:      Union[str, Any]=1_6, UpperCAmelCase__							:      Optional[Any]=6_4, UpperCAmelCase__							:      Optional[Any]=4_0_9_6, UpperCAmelCase__							:      Tuple=4, UpperCAmelCase__							:      List[str]=False, UpperCAmelCase__							:      Optional[int]=1_8, UpperCAmelCase__							:      int=1_6_0_0, UpperCAmelCase__							:      List[str]=1_0_0_0, UpperCAmelCase__							:      Union[str, Any]=True, UpperCAmelCase__							:      Union[str, Any]=True, UpperCAmelCase__							:      List[str]=0, UpperCAmelCase__							:      List[Any]=-1, UpperCAmelCase__							:      int=True, UpperCAmelCase__							:      Dict=0.1, UpperCAmelCase__							:      int=0.0, UpperCAmelCase__							:      str=True, UpperCAmelCase__							:      List[str]="normal", UpperCAmelCase__							:      List[str]=0.01, UpperCAmelCase__							:      Optional[int]=0.01, UpperCAmelCase__							:      List[str]=0.02, UpperCAmelCase__							:      Union[str, Any]=1E-5, UpperCAmelCase__							:      List[str]=0, **UpperCAmelCase__							:      Tuple, ):
    __lowercase								=						vocab_size
    __lowercase								=						[]
    self.cutoffs.extend(UpperCAmelCase__     )
    if proj_share_all_but_first:
       __lowercase								=						[False] + [True] * len(self.cutoffs     )
    else:
       __lowercase								=						[False] + [False] * len(self.cutoffs     )
    __lowercase								=						d_model
    __lowercase								=						d_embed
    __lowercase								=						d_head
    __lowercase								=						d_inner
    __lowercase								=						div_val
    __lowercase								=						pre_lnorm
    __lowercase								=						n_layer
    __lowercase								=						n_head
    __lowercase								=						mem_len
    __lowercase								=						same_length
    __lowercase								=						attn_type
    __lowercase								=						clamp_len
    __lowercase								=						sample_softmax
    __lowercase								=						adaptive
    __lowercase								=						dropout
    __lowercase								=						dropatt
    __lowercase								=						untie_r
    __lowercase								=						init
    __lowercase								=						init_range
    __lowercase								=						proj_init_std
    __lowercase								=						init_std
    __lowercase								=						layer_norm_epsilon
    super().__init__(eos_token_id=UpperCAmelCase__, **UpperCAmelCase__     )


 @property
 def  _lowercase			(      self							:      List[Any]     ):
    # Message copied from Transformer-XL documentation
    logger.info(F"""The model {self.model_type} is one of the few models that has no sequence length limit."""     )
    return -1



 @max_position_embeddings.setter
 def  _lowercase			(      self							:      List[str], UpperCAmelCase__							:      Optional[int]     ):
    # Message copied from Transformer-XL documentation
    raise NotImplementedError(
        F"""The model {self.model_type} is one of the few models that has no sequence length limit."""     )







"""simple docstring"""





import argparse
import shlex

import runhouse as rh


if __name__ == "__main__":
   # Refer to https://runhouse-docs.readthedocs-hosted.com/en/latest/api/python/cluster.html#hardware-setup for cloud access
   # setup instructions, if using on-demand hardware

   # If user passes --user <user> --host <host> --key_path <key_path> <example> <args>, fill them in as BYO cluster
   # If user passes --instance <instance> --provider <provider> <example> <args>, fill them in as on-demand cluster
   # Throw an error if user passes both BYO and on-demand cluster args
   # Otherwise, use default values
   _a							=							argparse.ArgumentParser()
   parser.add_argument('--user', type=str, default='ubuntu')
   parser.add_argument('--host', type=str, default='localhost')
   parser.add_argument('--key_path', type=str, default=None)
   parser.add_argument('--instance', type=str, default='V100:1')
   parser.add_argument('--provider', type=str, default='cheapest')
   parser.add_argument('--use_spot', type=bool, default=False)
   parser.add_argument('--example', type=str, default='pytorch/text-generation/run_generation.py')
   _a				,						_a							=							parser.parse_known_args()
   if args.host != "localhost":
      if args.instance != "V100:1" or args.provider != "cheapest":
         raise ValueError('Cannot specify both BYO and on-demand cluster args')
      _a							=							rh.cluster(
          name='rh-cluster', ips=[args.host], ssh_creds={'ssh_user': args.user, 'ssh_private_key': args.key_path}
      )
   else:
      _a							=							rh.cluster(
          name='rh-cluster', instance_type=args.instance, provider=args.provider, use_spot=args.use_spot
      )
   _a							=							args.example.rsplit('/', 1)[0]

   # Set up remote environment
   cluster.install_packages(['pip:./'])  # Installs transformers from local source
   # Note transformers is copied into the home directory on the remote machine, so we can install from there
   cluster.run([F"pip install -r transformers/examples/{example_dir}/requirements.txt"])
   cluster.run(['pip install torch --upgrade --extra-index-url https://download.pytorch.org/whl/cu117'])

   # Run example. You can bypass the CLI wrapper and paste your own code here.
   cluster.run([F"python transformers/examples/{args.example} {' '.join(shlex.quote(arg) for arg in unknown)}"])

   # Alternatively, we can just import and run a training function (especially if there's no wrapper CLI):
   # from my_script... import train
   # reqs = ['pip:./', 'torch', 'datasets', 'accelerate', 'evaluate', 'tqdm', 'scipy', 'scikit-learn', 'tensorboard']
   # launch_train_gpu = rh.function(fn=train,
   #                                system=gpu,
   #                                reqs=reqs,
   #                                name='train_bert_glue')
   #
   # We can pass in arguments just like we would to a function:
   # launch_train_gpu(num_epochs = 3, lr = 2e-5, seed = 42, batch_size = 16
   #                  stream_logs=True)




