


'''simple docstring'''



import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_lowercase      :      Dict				       =					logging.getLogger(__name__)

_lowercase      :      Optional[int]				       =					tf.data.AUTOTUNE


def 				lowerCamelCase__				(	):




						'''simple docstring'''






						UpperCAmelCase					    =     argparse.ArgumentParser(description='''Train a masked language model on TPU.'''					)
						parser.add_argument(
						    '''--pretrained_model_config'''				,       type=lowercase__				,       default='''roberta-base'''				,       help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!'''				,       )
						parser.add_argument(
						    '''--tokenizer'''				,       type=lowercase__				,       default='''unigram-tokenizer-wikitext'''				,       help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'''				,       )

						parser.add_argument(
						    '''--per_replica_batch_size'''				,       type=lowercase__				,       default=8				,       help='''Batch size per TPU core.'''				,       )

						parser.add_argument(
						    '''--no_tpu'''				,       action='''store_true'''				,       help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'''				,       )

						parser.add_argument(
						    '''--tpu_name'''				,       type=lowercase__				,       help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'''				,       default='''local'''				,       )

						parser.add_argument(
						    '''--tpu_zone'''				,       type=lowercase__				,       help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'''				,       )

						parser.add_argument(
						    '''--gcp_project'''				,       type=lowercase__				,       help='''Google cloud project name. Only used for non-Colab TPU nodes.'''					)

						parser.add_argument(
						    '''--bfloat16'''				,       action='''store_true'''				,       help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'''				,       )

						parser.add_argument(
						    '''--train_dataset'''				,       type=lowercase__				,       help='''Path to training dataset to load. If the path begins with `gs://`'''
						    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''				,       )

						parser.add_argument(
						    '''--shuffle_buffer_size'''				,       type=lowercase__				,       default=2**18				,       help='''Size of the shuffle buffer (in samples)'''				,       )

						parser.add_argument(
						    '''--eval_dataset'''				,       type=lowercase__				,       help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
						    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''				,       )

						parser.add_argument(
						    '''--num_epochs'''				,       type=lowercase__				,       default=1				,       help='''Number of epochs to train for.'''				,       )

						parser.add_argument(
						    '''--learning_rate'''				,       type=lowercase__				,       default=1E-4				,       help='''Learning rate to use for training.'''				,       )

						parser.add_argument(
						    '''--weight_decay_rate'''				,       type=lowercase__				,       default=1E-3				,       help='''Weight decay rate to use for training.'''				,       )

						parser.add_argument(
						    '''--max_length'''				,       type=lowercase__				,       default=5_12				,       help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'''				,       )

						parser.add_argument(
						    '''--mlm_probability'''				,       type=lowercase__				,       default=0.15				,       help='''Fraction of tokens to mask during training.'''				,       )

						parser.add_argument('''--output_dir'''				,       type=lowercase__				,       required=lowercase__				,       help='''Path to save model checkpoints to.'''					)
						parser.add_argument('''--hub_model_id'''				,       type=lowercase__				,       help='''Model ID to upload to on the Hugging Face Hub.'''					)

						UpperCAmelCase					    =     parser.parse_args()
						return args


def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						try:
												if args.tpu_name:
																		UpperCAmelCase					    =     tf.distribute.cluster_resolver.TPUClusterResolver(
																		    args.tpu_name				,       zone=args.tpu_zone				,       project=args.gcp_project					)
												else:
																		UpperCAmelCase					    =     tf.distribute.cluster_resolver.TPUClusterResolver()
						except ValueError:
												raise RuntimeError(
												    '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
												    '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''					)

						tf.config.experimental_connect_to_cluster(lowercase__					)
						tf.tpu.experimental.initialize_tpu_system(lowercase__					)

						return tpu


def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     0
						for file in file_list:
												UpperCAmelCase					    =     file.split('''/'''					)[-1]
												UpperCAmelCase					    =     re.search(R'''-\d+-(\d+)\.tfrecord'''				,       lowercase__					).group(1					)
												UpperCAmelCase					    =     int(lowercase__					)
												num_samples += sample_count

						return num_samples


def 				lowerCamelCase__				(	A       :      Optional[int]				,       A       :      Union[str, Any]				,       A       :      List[str]				,       A       :      int				,       A       :      Tuple				,       A       :      int=None					):




						'''simple docstring'''






						UpperCAmelCase					    =     count_samples(lowercase__					)
						UpperCAmelCase					    =     tf.data.Dataset.from_tensor_slices(lowercase__					)
						if shuffle:
												UpperCAmelCase					    =     dataset.shuffle(len(lowercase__					)					)
						UpperCAmelCase					    =     tf.data.TFRecordDataset(lowercase__				,       num_parallel_reads=lowercase__					)
						# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
						UpperCAmelCase					    =     dataset.apply(tf.data.experimental.assert_cardinality(lowercase__					)					)
						UpperCAmelCase					    =     dataset.map(lowercase__				,       num_parallel_calls=lowercase__					)
						if shuffle:
												assert shuffle_buffer_size is not None
												UpperCAmelCase					    =     dataset.shuffle(args.shuffle_buffer_size					)
						UpperCAmelCase					    =     dataset.batch(lowercase__				,       drop_remainder=lowercase__					)
						UpperCAmelCase					    =     dataset.map(lowercase__				,       num_parallel_calls=lowercase__					)
						UpperCAmelCase					    =     dataset.prefetch(lowercase__					)
						return dataset


def 				lowerCamelCase__				(	A       :      Optional[int]					):




						'''simple docstring'''






						if not args.no_tpu:
												UpperCAmelCase					    =     initialize_tpu(lowercase__					)
												UpperCAmelCase					    =     tf.distribute.TPUStrategy(lowercase__					)
						else:
												UpperCAmelCase					    =     tf.distribute.OneDeviceStrategy(device='''/gpu:0'''					)

						if args.bfloataa:
												tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''					)

						UpperCAmelCase					    =     AutoTokenizer.from_pretrained(args.tokenizer					)
						UpperCAmelCase					    =     AutoConfig.from_pretrained(args.pretrained_model_config					)
						UpperCAmelCase					    =     tokenizer.vocab_size

						UpperCAmelCase					    =     tf.io.gfile.glob(os.path.join(args.train_dataset				,       '''*.tfrecord'''					)					)
						if not training_records:
												raise ValueError(f"""No .tfrecord files found in {args.train_dataset}."""					)
						UpperCAmelCase					    =     tf.io.gfile.glob(os.path.join(args.eval_dataset				,       '''*.tfrecord'''					)					)
						if not eval_records:
												raise ValueError(f"""No .tfrecord files found in {args.eval_dataset}."""					)

						UpperCAmelCase					    =     count_samples(lowercase__					)

						UpperCAmelCase					    =     num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
						UpperCAmelCase					    =     steps_per_epoch * args.num_epochs

						with strategy.scope():
												UpperCAmelCase					    =     TFAutoModelForMaskedLM.from_config(lowercase__					)
												model(model.dummy_inputs					)  # Pass some dummy inputs through the model to ensure all the weights are built
												UpperCAmelCase    ,    UpperCAmelCase					    =     create_optimizer(
												    num_train_steps=lowercase__				,       num_warmup_steps=total_train_steps // 20				,       init_lr=args.learning_rate				,       weight_decay_rate=args.weight_decay_rate				,       )

												# Transformers models compute the right loss for their task by default when labels are passed, and will
												# use this for training unless you specify your own loss function in compile().
												model.compile(optimizer=lowercase__				,       metrics=['''accuracy''']					)

						def decode_fn(A       :      Any					):
												UpperCAmelCase					    =     {
												    '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa				,       shape=(args.max_length,)					),
												    '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa				,       shape=(args.max_length,)					),
												}
												return tf.io.parse_single_example(lowercase__				,       lowercase__					)

						# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
						# use their methods in our data pipeline.
						UpperCAmelCase					    =     DataCollatorForLanguageModeling(
						    tokenizer=lowercase__				,       mlm_probability=args.mlm_probability				,       mlm=lowercase__				,       return_tensors='''tf'''					)

						def mask_with_collator(A       :      Union[str, Any]					):
												# TF really needs an isin() function
												UpperCAmelCase					    =     (
												    ~tf.cast(batch['''attention_mask''']				,       tf.bool					)
												    | (batch['''input_ids'''] == tokenizer.cls_token_id)
												    | (batch['''input_ids'''] == tokenizer.sep_token_id)
												)
												UpperCAmelCase    ,    UpperCAmelCase					    =     data_collator.tf_mask_tokens(
												    batch['''input_ids''']				,       vocab_size=len(lowercase__					)				,       mask_token_id=tokenizer.mask_token_id				,       special_tokens_mask=lowercase__				,       )
												return batch

						UpperCAmelCase					    =     args.per_replica_batch_size * strategy.num_replicas_in_sync

						UpperCAmelCase					    =     prepare_dataset(
						    lowercase__				,       decode_fn=lowercase__				,       mask_fn=lowercase__				,       batch_size=lowercase__				,       shuffle=lowercase__				,       shuffle_buffer_size=args.shuffle_buffer_size				,       )

						UpperCAmelCase					    =     prepare_dataset(
						    lowercase__				,       decode_fn=lowercase__				,       mask_fn=lowercase__				,       batch_size=lowercase__				,       shuffle=lowercase__				,       )

						UpperCAmelCase					    =     []
						if args.hub_model_id:
												callbacks.append(
												    PushToHubCallback(output_dir=args.output_dir				,       hub_model_id=args.hub_model_id				,       tokenizer=lowercase__					)					)

						model.fit(
						    lowercase__				,       validation_data=lowercase__				,       epochs=args.num_epochs				,       callbacks=lowercase__				,       )

						model.save_pretrained(args.output_dir					)


if __name__ == "__main__":
	_lowercase      :      Optional[int]				       =					parse_args()
	main(args)









'''simple docstring'''



import os
import tempfile
import unittest
import uuid
from pathlib import Path

from transformers.testing_utils import get_tests_dir, require_soundfile, require_torch, require_vision
from transformers.tools.agent_types import AgentAudio, AgentImage, AgentText
from transformers.utils import is_soundfile_availble, is_torch_available, is_vision_available


if is_torch_available():
	import torch

if is_soundfile_availble():
	import soundfile as sf

if is_vision_available():
	from PIL import Image


def 				lowerCamelCase__				(	A       :      str=""					):




						'''simple docstring'''






						UpperCAmelCase					    =     tempfile.mkdtemp()
						return os.path.join(A				,       str(uuid.uuida()					) + suffix					)






@require_soundfile
@require_torch
class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			int				)->					int:

										"""simple docstring"""




										UpperCAmelCase					    =     torch.rand(12				,   dtype=torch.floataa				) - 0.5
										UpperCAmelCase					    =     AgentAudio(lowerCAmelCase				)
										UpperCAmelCase					    =     str(agent_type.to_string()				)

										# Ensure that the tensor and the agent_type's tensor are the same
										self.assertTrue(torch.allclose(lowerCAmelCase				,   agent_type.to_raw()				,   atol=1E-4				)				)

										del agent_type

										# Ensure the path remains even after the object deletion
										self.assertTrue(os.path.exists(lowerCAmelCase				)				)

										# Ensure that the file contains the same value as the original tensor
										UpperCAmelCase    ,    UpperCAmelCase					    =     sf.read(lowerCAmelCase				)
										self.assertTrue(torch.allclose(lowerCAmelCase				,   torch.tensor(lowerCAmelCase				)				,   atol=1E-4				)				)





				def 					a__(		self      :			Union[str, Any]				)->					Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     torch.rand(12				,   dtype=torch.floataa				) - 0.5
										UpperCAmelCase					    =     get_new_path(suffix='''.wav'''				)
										sf.write(lowerCAmelCase				,   lowerCAmelCase				,   16000				)

										UpperCAmelCase					    =     AgentAudio(lowerCAmelCase				)

										self.assertTrue(torch.allclose(lowerCAmelCase				,   agent_type.to_raw()				,   atol=1E-4				)				)
										self.assertEqual(agent_type.to_string()				,   lowerCAmelCase				)







@require_vision
@require_torch
class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			List[Any]				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     torch.randint(0				,   256				,   (64, 64, 3)				)
										UpperCAmelCase					    =     AgentImage(lowerCAmelCase				)
										UpperCAmelCase					    =     str(agent_type.to_string()				)

										# Ensure that the tensor and the agent_type's tensor are the same
										self.assertTrue(torch.allclose(lowerCAmelCase				,   agent_type._tensor				,   atol=1E-4				)				)

										self.assertIsInstance(agent_type.to_raw()				,   Image.Image				)

										# Ensure the path remains even after the object deletion
										del agent_type
										self.assertTrue(os.path.exists(lowerCAmelCase				)				)




				def 					a__(		self      :			List[Any]				)->					List[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     Path(get_tests_dir('''fixtures/tests_samples/COCO'''				)				) / '''000000039769.png'''
										UpperCAmelCase					    =     Image.open(lowerCAmelCase				)
										UpperCAmelCase					    =     AgentImage(lowerCAmelCase				)

										self.assertTrue(path.samefile(agent_type.to_string()				)				)
										self.assertTrue(image == agent_type.to_raw()				)

										# Ensure the path remains even after the object deletion
										del agent_type
										self.assertTrue(os.path.exists(lowerCAmelCase				)				)





				def 					a__(		self      :			Optional[Any]				)->					List[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     Path(get_tests_dir('''fixtures/tests_samples/COCO'''				)				) / '''000000039769.png'''
										UpperCAmelCase					    =     Image.open(lowerCAmelCase				)
										UpperCAmelCase					    =     AgentImage(lowerCAmelCase				)

										self.assertFalse(path.samefile(agent_type.to_string()				)				)
										self.assertTrue(image == agent_type.to_raw()				)

										# Ensure the path remains even after the object deletion
										del agent_type
										self.assertTrue(os.path.exists(lowerCAmelCase				)				)







class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			int				)->					Any:

										"""simple docstring"""




										UpperCAmelCase					    =     '''Hey!'''
										UpperCAmelCase					    =     AgentText(lowerCAmelCase				)

										self.assertEqual(lowerCAmelCase				,   agent_type.to_string()				)
										self.assertEqual(lowerCAmelCase				,   agent_type.to_raw()				)
										self.assertEqual(lowerCAmelCase				,   lowerCAmelCase				)





