import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
          CTRLForSequenceClassification,
          CTRLLMHeadModel,
          CTRLModel,
      )





class 					A							:


   def __init__(self					:						Optional[int]	,  __UpperCAmelCase					:						Dict	,  __UpperCAmelCase					:						Dict=1_4	,  __UpperCAmelCase					:						str=7	,  __UpperCAmelCase					:						List[str]=True	,  __UpperCAmelCase					:						Optional[int]=True	,  __UpperCAmelCase					:						List[Any]=True	,  __UpperCAmelCase					:						Union[str, Any]=True	,  __UpperCAmelCase					:						Dict=True	,  __UpperCAmelCase					:						Optional[Any]=9_9	,  __UpperCAmelCase					:						Tuple=3_2	,  __UpperCAmelCase					:						List[Any]=5	,  __UpperCAmelCase					:						str=4	,  __UpperCAmelCase					:						Union[str, Any]=3_7	,  __UpperCAmelCase					:						Optional[int]="gelu"	,  __UpperCAmelCase					:						Union[str, Any]=0.1	,  __UpperCAmelCase					:						List[Any]=0.1	,  __UpperCAmelCase					:						Optional[int]=5_1_2	,  __UpperCAmelCase					:						Optional[int]=1_6	,  __UpperCAmelCase					:						Optional[int]=2	,  __UpperCAmelCase					:						Tuple=0.02	,  __UpperCAmelCase					:						str=3	,  __UpperCAmelCase					:						str=4	,  __UpperCAmelCase					:						Optional[Any]=None	,  )						->    Any:
         """simple docstring"""



         UpperCAmelCase__								=  parent
         UpperCAmelCase__								=  batch_size
         UpperCAmelCase__								=  seq_length
         UpperCAmelCase__								=  is_training
         UpperCAmelCase__								=  use_token_type_ids
         UpperCAmelCase__								=  use_input_mask
         UpperCAmelCase__								=  use_labels
         UpperCAmelCase__								=  use_mc_token_ids
         UpperCAmelCase__								=  vocab_size
         UpperCAmelCase__								=  hidden_size
         UpperCAmelCase__								=  num_hidden_layers
         UpperCAmelCase__								=  num_attention_heads
         UpperCAmelCase__								=  intermediate_size
         UpperCAmelCase__								=  hidden_act
         UpperCAmelCase__								=  hidden_dropout_prob
         UpperCAmelCase__								=  attention_probs_dropout_prob
         UpperCAmelCase__								=  max_position_embeddings
         UpperCAmelCase__								=  type_vocab_size
         UpperCAmelCase__								=  type_sequence_label_size
         UpperCAmelCase__								=  initializer_range
         UpperCAmelCase__								=  num_labels
         UpperCAmelCase__								=  num_choices
         UpperCAmelCase__								=  scope
         UpperCAmelCase__								=  self.vocab_size - 1


   def       lowercase_				(self					:						str		)						->    Any:
         """simple docstring"""



         UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	,  self.vocab_size		)

         UpperCAmelCase__								=  None
         if self.use_input_mask:
               UpperCAmelCase__								=  random_attention_mask([self.batch_size, self.seq_length]		)

         UpperCAmelCase__								=  None
         if self.use_token_type_ids:
               UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	,  self.type_vocab_size		)

         UpperCAmelCase__								=  None
         if self.use_mc_token_ids:
               UpperCAmelCase__								=  ids_tensor([self.batch_size, self.num_choices]	,  self.seq_length		)

         UpperCAmelCase__								=  None
         UpperCAmelCase__								=  None
         UpperCAmelCase__								=  None
         if self.use_labels:
               UpperCAmelCase__								=  ids_tensor([self.batch_size]	,  self.type_sequence_label_size		)
               UpperCAmelCase__								=  ids_tensor([self.batch_size, self.seq_length]	,  self.num_labels		)
               UpperCAmelCase__								=  ids_tensor([self.batch_size]	,  self.num_choices		)

         UpperCAmelCase__								=  self.get_config()

         UpperCAmelCase__								=  ids_tensor([self.num_hidden_layers, self.num_attention_heads]	,  2		)

         return (
             config,
             input_ids,
             input_mask,
             head_mask,
             token_type_ids,
             mc_token_ids,
             sequence_labels,
             token_labels,
             choice_labels,
         )


   def       lowercase_				(self					:						Optional[int]		)						->    Optional[Any]:
         """simple docstring"""



         return CTRLConfig(
             vocab_size=self.vocab_size	,  n_embd=self.hidden_size	,  n_layer=self.num_hidden_layers	,  n_head=self.num_attention_heads	,  n_positions=self.max_position_embeddings	,  pad_token_id=self.pad_token_id	,  )


   def       lowercase_				(self					:						Tuple	,  __UpperCAmelCase					:						Optional[Any]	,  __UpperCAmelCase					:						Dict	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						Union[str, Any]	,  __UpperCAmelCase					:						str	,  *__UpperCAmelCase					:						List[str]		)						->    Dict:
         """simple docstring"""



         UpperCAmelCase__								=  CTRLModel(config=__UpperCAmelCase		)
         model.to(__UpperCAmelCase		)
         model.eval()

         model(__UpperCAmelCase	,  token_type_ids=__UpperCAmelCase	,  head_mask=__UpperCAmelCase		)
         model(__UpperCAmelCase	,  token_type_ids=__UpperCAmelCase		)
         UpperCAmelCase__								=  model(__UpperCAmelCase		)
         self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, self.seq_length, self.hidden_size)		)
         self.parent.assertEqual(len(result.past_key_values		)	,  config.n_layer		)


   def       lowercase_				(self					:						Any	,  __UpperCAmelCase					:						str	,  __UpperCAmelCase					:						Tuple	,  __UpperCAmelCase					:						Dict	,  __UpperCAmelCase					:						Union[str, Any]	,  __UpperCAmelCase					:						Optional[int]	,  *__UpperCAmelCase					:						Dict		)						->    Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  CTRLLMHeadModel(__UpperCAmelCase		)
         model.to(__UpperCAmelCase		)
         model.eval()

         UpperCAmelCase__								=  model(__UpperCAmelCase	,  token_type_ids=__UpperCAmelCase	,  labels=__UpperCAmelCase		)
         self.parent.assertEqual(result.loss.shape	,  ()		)
         self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.seq_length, self.vocab_size)		)


   def       lowercase_				(self					:						Optional[Any]		)						->    int:
         """simple docstring"""



         UpperCAmelCase__								=  self.prepare_config_and_inputs()

         (
             (
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				(
             UpperCAmelCase__
         )		,				
         )								=  config_and_inputs

         UpperCAmelCase__								=  {"input_ids": input_ids, "token_type_ids": token_type_ids, "head_mask": head_mask}

         return config, inputs_dict





   def       lowercase_				(self					:						Dict	,  __UpperCAmelCase					:						Union[str, Any]	,  __UpperCAmelCase					:						Any	,  __UpperCAmelCase					:						Optional[Any]	,  __UpperCAmelCase					:						Union[str, Any]	,  *__UpperCAmelCase					:						Optional[Any]		)						->    List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.num_labels
         UpperCAmelCase__								=  CTRLForSequenceClassification(__UpperCAmelCase		)
         model.to(__UpperCAmelCase		)
         model.eval()
         UpperCAmelCase__								=  ids_tensor([self.batch_size]	,  self.type_sequence_label_size		)
         UpperCAmelCase__								=  model(__UpperCAmelCase	,  token_type_ids=__UpperCAmelCase	,  labels=__UpperCAmelCase		)
         self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.num_labels)		)





@require_torch
class 					A							(    UpperCAmelCase_		,    UpperCAmelCase_		,    UpperCAmelCase_		,    unittest.TestCase					):
   __UpperCAmelCase			:			Tuple              =      (CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
   __UpperCAmelCase			:			List[Any]              =      (CTRLLMHeadModel,) if is_torch_available() else ()
   __UpperCAmelCase			:			Tuple              =      (
       {
           'feature-extraction': CTRLModel,
           'text-classification': CTRLForSequenceClassification,
           'text-generation': CTRLLMHeadModel,
           'zero-shot': CTRLForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   __UpperCAmelCase			:			List[str]              =      True
   __UpperCAmelCase			:			Dict              =      False
   __UpperCAmelCase			:			Dict              =      False


   def       lowercase_				(self					:						int	,  __UpperCAmelCase					:						Tuple	,  __UpperCAmelCase					:						Dict	,  __UpperCAmelCase					:						Union[str, Any]	,  __UpperCAmelCase					:						str	,  __UpperCAmelCase					:						int		)						->    Any:
         """simple docstring"""



         if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
               # config could not be created.
               return True

         return False


   def       lowercase_				(self					:						List[Any]		)						->    str:
         """simple docstring"""



         UpperCAmelCase__								=  CTRLModelTester(self		)
         UpperCAmelCase__								=  ConfigTester(self	,  config_class=__UpperCAmelCase	,  n_embd=3_7		)


   def       lowercase_				(self					:						List[str]		)						->    Union[str, Any]:
         """simple docstring"""



         super().tearDown()
         # clean-up as much as possible GPU memory occupied by PyTorch
         gc.collect()
         torch.cuda.empty_cache()


   def       lowercase_				(self					:						Optional[Any]		)						->    List[Any]:
         """simple docstring"""



         self.config_tester.run_common_tests()


   def       lowercase_				(self					:						Optional[int]		)						->    Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_ctrl_model(*__UpperCAmelCase		)


   def       lowercase_				(self					:						Union[str, Any]		)						->    Optional[int]:
         """simple docstring"""



         UpperCAmelCase__								=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head_model(*__UpperCAmelCase		)


   @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."		)
   def       lowercase_				(self					:						List[str]		)						->    Optional[int]:
         """simple docstring"""



         pass


   @slow
   def       lowercase_				(self					:						Any		)						->    Any:
         """simple docstring"""



         for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase__								=  CTRLModel.from_pretrained(__UpperCAmelCase		)
               self.assertIsNotNone(__UpperCAmelCase		)





   @unittest.skip("The model doesn't support left padding"		)  # and it's not used enough to be worth fixing :)
   def       lowercase_				(self					:						Tuple		)						->    List[Any]:
         """simple docstring"""



         pass





@require_torch
class 					A							(    unittest.TestCase					):


   def       lowercase_				(self					:						Any		)						->    List[str]:
         """simple docstring"""



         super().tearDown()
         # clean-up as much as possible GPU memory occupied by PyTorch
         gc.collect()
         torch.cuda.empty_cache()





   @slow
   def       lowercase_				(self					:						List[str]		)						->    Optional[int]:
         """simple docstring"""



         UpperCAmelCase__								=  CTRLLMHeadModel.from_pretrained("ctrl"		)
         model.to(__UpperCAmelCase		)
         UpperCAmelCase__								=  torch.tensor(
             [[1_1_8_5_9, 0, 1_6_1_1, 8]]	,  dtype=torch.long	,  device=__UpperCAmelCase		)  # Legal the president is
         UpperCAmelCase__								=  [
             1_1_8_5_9,
             0,
             1_6_1_1,
             8,
             5,
             1_5_0,
             2_6_4_4_9,
             2,
             1_9,
             3_4_8,
             4_6_9,
             3,
             2_5_9_5,
             4_8,
             2_0_7_4_0,
             2_4_6_5_3_3,
             2_4_6_5_3_3,
             1_9,
             3_0,
             5,
         ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

         UpperCAmelCase__								=  model.generate(__UpperCAmelCase	,  do_sample=__UpperCAmelCase		)
         self.assertListEqual(output_ids[0].tolist()	,  __UpperCAmelCase		)




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__    							= logging.getLogger(__name__)




def        lowerCAmelCase_			(					)					->    Union[str, Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  argparse.ArgumentParser(
          description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
      parser.add_argument(
          "--dataset_name",  type=__A,  default="wikitext",  help="Name of the training. Explore datasets at: hf.co/datasets.",  )
      parser.add_argument(
          "--dataset_config",  type=__A,  default="wikitext-103-raw-v1",  help="Configuration name of the dataset."							)
      parser.add_argument(
          "--tokenizer_name_or_path",  type=__A,  default="sayakpaul/unigram-tokenizer-wikitext",  help="Tokenizer identifier. Can be a local filepath or a Hub identifier.",  )
      parser.add_argument(
          "--shard_size",  type=__A,  default=1_000,  help="Number of entries to go in a single shard.",  )
      parser.add_argument("--split",  type=__A,  default="train",  choices=["train", "test", "validation"]							)
      parser.add_argument(
          "--limit",  default=__A,  type=__A,  help="Limit the number of shards (used for debugging).",  )
      parser.add_argument(
          "--max_length",  type=__A,  default=512,  help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
          " sequence length that is a multiple of 8.",  )
      parser.add_argument(
          "--output_dir",  default="tf-tpu",  type=__A,  help="Output directory where the TFRecord shards will be saved. If the"
          " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
          " shards will be directly saved to a Google Cloud Storage bucket.",  )

      UpperCAmelCase__								=  parser.parse_args()
      return args




def        lowerCAmelCase_			(					__A							)					->    Optional[int]:







      '''simple docstring'''


      def fn(__A							):
            return tokenizer(examples["text"]							)

      return fn




def        lowerCAmelCase_			(					__A							)					->    List[Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  []
      for i in range(len(tokenized_data["input_ids"]							)							):
            UpperCAmelCase__								=  {
                "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
                "attention_mask": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
            }
            UpperCAmelCase__								=  tf.train.Features(feature=__A							)
            UpperCAmelCase__								=  tf.train.Example(features=__A							)
            UpperCAmelCase__								=  example.SerializeToString()
            records.append(__A							)
      return records




def        lowerCAmelCase_			(					__A							)					->    Union[str, Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  datasets.load_dataset(args.dataset_name,  args.dataset_config,  split=args.split							)

      if args.limit is not None:
            UpperCAmelCase__								=  min(len(__A							),  args.limit							)
            UpperCAmelCase__								=  dataset.select(range(__A							)							)
            print(f"""Limiting the dataset to {args.limit} entries."""							)

      UpperCAmelCase__								=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir							):
                  os.makedirs(args.output_dir							)
            UpperCAmelCase__								=  os.path.join(args.output_dir,  args.split							)
            if not os.path.exists(__A							):
                  os.makedirs(__A							)
      else:
            UpperCAmelCase__								=  os.path.join(args.output_dir,  args.split							)

      # Tokenize the whole dataset at once.
      UpperCAmelCase__								=  tokenize_function(__A							)
      UpperCAmelCase__								=  dataset.map(__A,  batched=__A,  num_proc=4,  remove_columns=["text"]							)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(__A							):
            # Concatenate all texts.
            UpperCAmelCase__								=  {k: sum(examples[k],  []							) for k in examples.keys()}
            UpperCAmelCase__								=  len(concatenated_examples[list(examples.keys()							)[0]]							)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            UpperCAmelCase__								=  (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            UpperCAmelCase__								=  {
                k: [t[i : i + args.max_length] for i in range(0,  __A,  args.max_length							)]
                for k, t in concatenated_examples.items()
            }
            return result

      UpperCAmelCase__								=  dataset_tokenized.map(__A,  batched=__A,  batch_size=1_000,  num_proc=4							)

      UpperCAmelCase__								=  0
      UpperCAmelCase__								=  0
      for shard in range(0,  len(__A							),  args.shard_size							):
            UpperCAmelCase__								=  grouped_dataset[shard : shard + args.shard_size]
            UpperCAmelCase__								=  len(dataset_snapshot["input_ids"]							)
            UpperCAmelCase__								=  os.path.join(__A,  f"""dataset-{shard_count}-{records_containing}.tfrecord"""							)
            UpperCAmelCase__								=  get_serialized_examples(__A							)

            with tf.io.TFRecordWriter(__A							) as out_file:
                  for i in range(len(__A							)							):
                        UpperCAmelCase__								=  serialized_examples[i]
                        out_file.write(__A							)
                  print("Wrote file {} containing {} records".format(__A,  __A							)							)

            shard_count += 1
            total_records += records_containing

      with open(f"""split-{args.split}-records-count.txt""",  "w"							) as f:
            print(f"""Total {args.split} records: {total_records}""",  file=__A							)


if __name__ == "__main__":
      UpperCamelCase__    							= parse_args()
      main(args)



