
"""simple docstring"""





import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_a														=		logging.getLogger(__name__)

_a														=		tf.data.AUTOTUNE



def 				lowerCamelCase__		(      )							->		Optional[Any]:


					"""simple docstring"""





					_UpperCamelCase										=  argparse.ArgumentParser(description='''Train a masked language model on TPU.'''     )
					parser.add_argument(
					    '''--pretrained_model_config''',					type=lowerCamelCase__,					default='''roberta-base''',					help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!''',					)
					parser.add_argument(
					    '''--tokenizer''',					type=lowerCamelCase__,					default='''unigram-tokenizer-wikitext''',					help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.''',					)

					parser.add_argument(
					    '''--per_replica_batch_size''',					type=lowerCamelCase__,					default=8,					help='''Batch size per TPU core.''',					)

					parser.add_argument(
					    '''--no_tpu''',					action='''store_true''',					help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.''',					)

					parser.add_argument(
					    '''--tpu_name''',					type=lowerCamelCase__,					help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.''',					default='''local''',					)

					parser.add_argument(
					    '''--tpu_zone''',					type=lowerCamelCase__,					help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.''',					)

					parser.add_argument(
					    '''--gcp_project''',					type=lowerCamelCase__,					help='''Google cloud project name. Only used for non-Colab TPU nodes.'''     )

					parser.add_argument(
					    '''--bfloat16''',					action='''store_true''',					help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.''',					)

					parser.add_argument(
					    '''--train_dataset''',					type=lowerCamelCase__,					help='''Path to training dataset to load. If the path begins with `gs://`'''
					    ''' then the dataset will be loaded from a Google Cloud Storage bucket.''',					)

					parser.add_argument(
					    '''--shuffle_buffer_size''',					type=lowerCamelCase__,					default=2**18,					help='''Size of the shuffle buffer (in samples)''',					)

					parser.add_argument(
					    '''--eval_dataset''',					type=lowerCamelCase__,					help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
					    ''' then the dataset will be loaded from a Google Cloud Storage bucket.''',					)

					parser.add_argument(
					    '''--num_epochs''',					type=lowerCamelCase__,					default=1,					help='''Number of epochs to train for.''',					)

					parser.add_argument(
					    '''--learning_rate''',					type=lowerCamelCase__,					default=1e-4,					help='''Learning rate to use for training.''',					)

					parser.add_argument(
					    '''--weight_decay_rate''',					type=lowerCamelCase__,					default=1e-3,					help='''Weight decay rate to use for training.''',					)

					parser.add_argument(
					    '''--max_length''',					type=lowerCamelCase__,					default=5_12,					help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py''',					)

					parser.add_argument(
					    '''--mlm_probability''',					type=lowerCamelCase__,					default=0.15,					help='''Fraction of tokens to mask during training.''',					)

					parser.add_argument('''--output_dir''',					type=lowerCamelCase__,					required=lowerCamelCase__,					help='''Path to save model checkpoints to.'''     )
					parser.add_argument('''--hub_model_id''',					type=lowerCamelCase__,					help='''Model ID to upload to on the Hugging Face Hub.'''     )

					_UpperCamelCase										=  parser.parse_args()
					return args



def 				lowerCamelCase__		(      __snake_case     )							->		List[Any]:


					"""simple docstring"""





					try:
										if args.tpu_name:
															_UpperCamelCase										=  tf.distribute.cluster_resolver.TPUClusterResolver(
															    args.tpu_name,					zone=args.tpu_zone,					project=args.gcp_project     )
										else:
															_UpperCamelCase										=  tf.distribute.cluster_resolver.TPUClusterResolver()
					except ValueError:
										raise RuntimeError(
										    '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
										    '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''     )

					tf.config.experimental_connect_to_cluster(lowerCamelCase__     )
					tf.tpu.experimental.initialize_tpu_system(lowerCamelCase__     )

					return tpu



def 				lowerCamelCase__		(      __snake_case     )							->		str:


					"""simple docstring"""





					_UpperCamelCase										=  0
					for file in file_list:
										_UpperCamelCase										=  file.split('''/'''     )[-1]
										_UpperCamelCase										=  re.search(r'''-\d+-(\d+)\.tfrecord''',					lowerCamelCase__     ).group(1     )
										_UpperCamelCase										=  int(lowerCamelCase__     )
										num_samples += sample_count

					return num_samples



def 				lowerCamelCase__		(      __snake_case,					__snake_case,					__snake_case,					__snake_case,					__snake_case,					__snake_case=None     )							->		List[Any]:


					"""simple docstring"""





					_UpperCamelCase										=  count_samples(lowerCamelCase__     )
					_UpperCamelCase										=  tf.data.Dataset.from_tensor_slices(lowerCamelCase__     )
					if shuffle:
										_UpperCamelCase										=  dataset.shuffle(len(lowerCamelCase__     )     )
					_UpperCamelCase										=  tf.data.TFRecordDataset(lowerCamelCase__,					num_parallel_reads=lowerCamelCase__     )
					# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
					_UpperCamelCase										=  dataset.apply(tf.data.experimental.assert_cardinality(lowerCamelCase__     )     )
					_UpperCamelCase										=  dataset.map(lowerCamelCase__,					num_parallel_calls=lowerCamelCase__     )
					if shuffle:
										assert shuffle_buffer_size is not None
										_UpperCamelCase										=  dataset.shuffle(args.shuffle_buffer_size     )
					_UpperCamelCase										=  dataset.batch(lowerCamelCase__,					drop_remainder=lowerCamelCase__     )
					_UpperCamelCase										=  dataset.map(lowerCamelCase__,					num_parallel_calls=lowerCamelCase__     )
					_UpperCamelCase										=  dataset.prefetch(lowerCamelCase__     )
					return dataset



def 				lowerCamelCase__		(      __snake_case     )							->		Dict:


					"""simple docstring"""





					if not args.no_tpu:
										_UpperCamelCase										=  initialize_tpu(lowerCamelCase__     )
										_UpperCamelCase										=  tf.distribute.TPUStrategy(lowerCamelCase__     )
					else:
										_UpperCamelCase										=  tf.distribute.OneDeviceStrategy(device='''/gpu:0'''     )

					if args.bfloataa:
										tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''     )

					_UpperCamelCase										=  AutoTokenizer.from_pretrained(args.tokenizer     )
					_UpperCamelCase										=  AutoConfig.from_pretrained(args.pretrained_model_config     )
					_UpperCamelCase										=  tokenizer.vocab_size

					_UpperCamelCase										=  tf.io.gfile.glob(os.path.join(args.train_dataset,					'''*.tfrecord'''     )     )
					if not training_records:
										raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.'''     )
					_UpperCamelCase										=  tf.io.gfile.glob(os.path.join(args.eval_dataset,					'''*.tfrecord'''     )     )
					if not eval_records:
										raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.'''     )

					_UpperCamelCase										=  count_samples(lowerCamelCase__     )

					_UpperCamelCase										=  num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
					_UpperCamelCase										=  steps_per_epoch * args.num_epochs

					with strategy.scope():
										_UpperCamelCase										=  TFAutoModelForMaskedLM.from_config(lowerCamelCase__     )
										model(model.dummy_inputs     )  # Pass some dummy inputs through the model to ensure all the weights are built
										_UpperCamelCase			,  _UpperCamelCase										=  create_optimizer(
										    num_train_steps=lowerCamelCase__,					num_warmup_steps=total_train_steps // 20,					init_lr=args.learning_rate,					weight_decay_rate=args.weight_decay_rate,					)

										# Transformers models compute the right loss for their task by default when labels are passed, and will
										# use this for training unless you specify your own loss function in compile().
										model.compile(optimizer=lowerCamelCase__,					metrics=['''accuracy''']     )

					def decode_fn(__snake_case     ):
										_UpperCamelCase										=  {
										    '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa,					shape=(args.max_length,)     ),
										    '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa,					shape=(args.max_length,)     ),
										}
										return tf.io.parse_single_example(lowerCamelCase__,					lowerCamelCase__     )

					# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
					# use their methods in our data pipeline.
					_UpperCamelCase										=  DataCollatorForLanguageModeling(
					    tokenizer=lowerCamelCase__,					mlm_probability=args.mlm_probability,					mlm=lowerCamelCase__,					return_tensors='''tf'''     )

					def mask_with_collator(__snake_case     ):
										# TF really needs an isin() function
										_UpperCamelCase										=  (
										    ~tf.cast(batch['''attention_mask'''],					tf.bool     )
										    | (batch['''input_ids'''] == tokenizer.cls_token_id)
										    | (batch['''input_ids'''] == tokenizer.sep_token_id)
										)
										_UpperCamelCase			,  _UpperCamelCase										=  data_collator.tf_mask_tokens(
										    batch['''input_ids'''],					vocab_size=len(lowerCamelCase__     ),					mask_token_id=tokenizer.mask_token_id,					special_tokens_mask=lowerCamelCase__,					)
										return batch

					_UpperCamelCase										=  args.per_replica_batch_size * strategy.num_replicas_in_sync

					_UpperCamelCase										=  prepare_dataset(
					    lowerCamelCase__,					decode_fn=lowerCamelCase__,					mask_fn=lowerCamelCase__,					batch_size=lowerCamelCase__,					shuffle=lowerCamelCase__,					shuffle_buffer_size=args.shuffle_buffer_size,					)

					_UpperCamelCase										=  prepare_dataset(
					    lowerCamelCase__,					decode_fn=lowerCamelCase__,					mask_fn=lowerCamelCase__,					batch_size=lowerCamelCase__,					shuffle=lowerCamelCase__,					)

					_UpperCamelCase										=  []
					if args.hub_model_id:
										callbacks.append(
										    PushToHubCallback(output_dir=args.output_dir,					hub_model_id=args.hub_model_id,					tokenizer=lowerCamelCase__     )     )

					model.fit(
					    lowerCamelCase__,					validation_data=lowerCamelCase__,					epochs=args.num_epochs,					callbacks=lowerCamelCase__,					)

					model.save_pretrained(args.output_dir     )


if __name__ == "__main__":
				_a														=		parse_args()
				main(args)








"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """facebook/xlm-roberta-xl""": """https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json""",
    """facebook/xlm-roberta-xxl""": """https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json""",
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}






class    _UpperCAmelCase(   lowerCamelCase     ):
		lowercase__      =				'xlm-roberta-xl'



		def __init__( self    ,		__a=25_08_80    ,		__a=25_60    ,		__a=36    ,		__a=32    ,		__a=1_02_40    ,		__a="gelu"    ,		__a=0.1    ,		__a=0.1    ,		__a=5_14    ,		__a=1    ,		__a=0.02    ,		__a=1e-05    ,		__a=1    ,		__a=0    ,		__a=2    ,		__a="absolute"    ,		__a=True    ,		__a=None    ,		**__a    ,		)	->	Tuple:






							'''simple docstring'''
							super().__init__(pad_token_id=__a    ,		bos_token_id=__a    ,		eos_token_id=__a    ,		**__a)
							_UpperCamelCase										=  vocab_size
							_UpperCamelCase										=  hidden_size
							_UpperCamelCase										=  num_hidden_layers
							_UpperCamelCase										=  num_attention_heads
							_UpperCamelCase										=  hidden_act
							_UpperCamelCase										=  intermediate_size
							_UpperCamelCase										=  hidden_dropout_prob
							_UpperCamelCase										=  attention_probs_dropout_prob
							_UpperCamelCase										=  max_position_embeddings
							_UpperCamelCase										=  type_vocab_size
							_UpperCamelCase										=  initializer_range
							_UpperCamelCase										=  layer_norm_eps
							_UpperCamelCase										=  position_embedding_type
							_UpperCamelCase										=  use_cache
							_UpperCamelCase										=  classifier_dropout






class    _UpperCAmelCase(   lowerCamelCase     ):



		@property
		def 						UpperCAmelCase							( self)	->	Mapping[str, Mapping[int, str]]:






							'''simple docstring'''
							if self.task == "multiple-choice":
												_UpperCamelCase										=  {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
							else:
												_UpperCamelCase										=  {0: '''batch''', 1: '''sequence'''}
							return OrderedDict(
							    [
							        ('''input_ids''', dynamic_axis),
							        ('''attention_mask''', dynamic_axis),
							    ])






