


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_A			=						logging.getLogger(__name__)

def 						_UpperCAmelCase    (      ):
     __UpperCamelCase   							=argparse.ArgumentParser(
         description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'    )
     parser.add_argument(
         '--dataset_name'      ,					type=SCREAMING_SNAKE_CASE__      ,					default='wikitext'      ,					help='Name of the training. Explore datasets at: hf.co/datasets.'      ,					)
     parser.add_argument(
         '--dataset_config'      ,					type=SCREAMING_SNAKE_CASE__      ,					default='wikitext-103-raw-v1'      ,					help='Configuration name of the dataset.'    )
     parser.add_argument(
         '--tokenizer_name_or_path'      ,					type=SCREAMING_SNAKE_CASE__      ,					default='sayakpaul/unigram-tokenizer-wikitext'      ,					help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'      ,					)
     parser.add_argument(
         '--shard_size'      ,					type=SCREAMING_SNAKE_CASE__      ,					default=10_00      ,					help='Number of entries to go in a single shard.'      ,					)
     parser.add_argument('--split'      ,					type=SCREAMING_SNAKE_CASE__      ,					default='train'      ,					choices=['train', 'test', 'validation']    )
     parser.add_argument(
         '--limit'      ,					default=SCREAMING_SNAKE_CASE__      ,					type=SCREAMING_SNAKE_CASE__      ,					help='Limit the number of shards (used for debugging).'      ,					)
     parser.add_argument(
         '--max_length'      ,					type=SCREAMING_SNAKE_CASE__      ,					default=5_12      ,					help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
         ' sequence length that is a multiple of 8.'      ,					)
     parser.add_argument(
         '--output_dir'      ,					default='tf-tpu'      ,					type=SCREAMING_SNAKE_CASE__      ,					help='Output directory where the TFRecord shards will be saved. If the'
         ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
         ' shards will be directly saved to a Google Cloud Storage bucket.'      ,					)

     __UpperCamelCase   							=parser.parse_args()
     return args

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Optional[Any]    ):
     def fn(SCREAMING_SNAKE_CASE__      : Union[str, Any]    ):
          return tokenizer(examples['text']    )

     return fn

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Any    ):
     __UpperCamelCase   							=[]
     for i in range(len(tokenized_data['input_ids']    )    ):
          __UpperCamelCase   							={
              'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]    )    ),
              'attention_mask': tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]    )    ),
          }
          __UpperCamelCase   							=tf.train.Features(feature=SCREAMING_SNAKE_CASE__    )
          __UpperCamelCase   							=tf.train.Example(features=SCREAMING_SNAKE_CASE__    )
          __UpperCamelCase   							=example.SerializeToString()
          records.append(SCREAMING_SNAKE_CASE__    )
     return records

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Any    ):
     __UpperCamelCase   							=datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split    )

     if args.limit is not None:
          __UpperCamelCase   							=min(len(SCREAMING_SNAKE_CASE__    )      ,					args.limit    )
          __UpperCamelCase   							=dataset.select(range(SCREAMING_SNAKE_CASE__    )    )
          print(F'Limiting the dataset to {args.limit} entries.'    )

     __UpperCamelCase   							=AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir    ):
               os.makedirs(args.output_dir    )
          __UpperCamelCase   							=os.path.join(args.output_dir      ,					args.split    )
          if not os.path.exists(SCREAMING_SNAKE_CASE__    ):
               os.makedirs(SCREAMING_SNAKE_CASE__    )
     else:
          __UpperCamelCase   							=os.path.join(args.output_dir      ,					args.split    )

     # Tokenize the whole dataset at once.
     __UpperCamelCase   							=tokenize_function(SCREAMING_SNAKE_CASE__    )
     __UpperCamelCase   							=dataset.map(SCREAMING_SNAKE_CASE__      ,					batched=SCREAMING_SNAKE_CASE__      ,					num_proc=4      ,					remove_columns=['text']    )

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(SCREAMING_SNAKE_CASE__      : str    ):
          # Concatenate all texts.
          __UpperCamelCase   							={k: sum(examples[k]      ,					[]    ) for k in examples.keys()}
          __UpperCamelCase   							=len(concatenated_examples[list(examples.keys()    )[0]]    )
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          __UpperCamelCase   							=(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          __UpperCamelCase   							={
              k: [t[i : i + args.max_length] for i in range(0      ,					SCREAMING_SNAKE_CASE__      ,					args.max_length    )]
              for k, t in concatenated_examples.items()
          }
          return result

     __UpperCamelCase   							=dataset_tokenized.map(SCREAMING_SNAKE_CASE__      ,					batched=SCREAMING_SNAKE_CASE__      ,					batch_size=10_00      ,					num_proc=4    )

     __UpperCamelCase   							=0
     __UpperCamelCase   							=0
     for shard in range(0      ,					len(SCREAMING_SNAKE_CASE__    )      ,					args.shard_size    ):
          __UpperCamelCase   							=grouped_dataset[shard : shard + args.shard_size]
          __UpperCamelCase   							=len(dataset_snapshot['input_ids']    )
          __UpperCamelCase   							=os.path.join(SCREAMING_SNAKE_CASE__      ,					F'dataset-{shard_count}-{records_containing}.tfrecord'    )
          __UpperCamelCase   							=get_serialized_examples(SCREAMING_SNAKE_CASE__    )

          with tf.io.TFRecordWriter(SCREAMING_SNAKE_CASE__    ) as out_file:
               for i in range(len(SCREAMING_SNAKE_CASE__    )    ):
                    __UpperCamelCase   							=serialized_examples[i]
                    out_file.write(SCREAMING_SNAKE_CASE__    )
               print('Wrote file {} containing {} records'.format(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    )    )

          shard_count += 1
          total_records += records_containing

     with open(F'split-{args.split}-records-count.txt'      ,					'w'    ) as f:
          print(F'Total {args.split} records: {total_records}'      ,					file=SCREAMING_SNAKE_CASE__    )


if __name__ == "__main__":
   _A			=						parse_args()
   main(args)









import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Optional[int]              =  "MCTCTFeatureExtractor"
    UpperCAmelCase__						:		str              =  "AutoTokenizer"






    def __init__(      self ,       A_ ,       A_   )  ->	Dict:
         super().__init__(A_ ,       A_   )
         __UpperCamelCase   							=self.feature_extractor
         __UpperCamelCase   							=False






    def __call__(      self ,       *A_ ,       **A_   )  ->	Optional[Any]:
         # For backward compatibility
         if self._in_target_context_manager:
              return self.current_processor(*A_ ,       **A_   )

         if "raw_speech" in kwargs:
              warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'   )
              __UpperCamelCase   							=kwargs.pop('raw_speech'   )
         else:
              __UpperCamelCase   							=kwargs.pop('audio' ,       A_   )
         __UpperCamelCase   							=kwargs.pop('sampling_rate' ,       A_   )
         __UpperCamelCase   							=kwargs.pop('text' ,       A_   )
         if len(A_   ) > 0:
              __UpperCamelCase   							=args[0]
              __UpperCamelCase   							=args[1:]

         if audio is None and text is None:
              raise ValueError('You need to specify either an `audio` or `text` input to process.'   )

         if audio is not None:
              __UpperCamelCase   							=self.feature_extractor(A_ ,       *A_ ,       sampling_rate=A_ ,       **A_   )
         if text is not None:
              __UpperCamelCase   							=self.tokenizer(A_ ,       **A_   )

         if text is None:
              return inputs
         elif audio is None:
              return encodings
         else:
              __UpperCamelCase   							=encodings['input_ids']
              return inputs






    def _a (      self ,       *A_ ,       **A_   )  ->	str:
         return self.tokenizer.batch_decode(*A_ ,       **A_   )






    def _a (      self ,       *A_ ,       **A_   )  ->	List[Any]:
         # For backward compatibility
         if self._in_target_context_manager:
              return self.current_processor.pad(*A_ ,       **A_   )

         __UpperCamelCase   							=kwargs.pop('input_features' ,       A_   )
         __UpperCamelCase   							=kwargs.pop('labels' ,       A_   )
         if len(A_   ) > 0:
              __UpperCamelCase   							=args[0]
              __UpperCamelCase   							=args[1:]

         if input_features is not None:
              __UpperCamelCase   							=self.feature_extractor.pad(A_ ,       *A_ ,       **A_   )
         if labels is not None:
              __UpperCamelCase   							=self.tokenizer.pad(A_ ,       **A_   )

         if labels is None:
              return input_features
         elif input_features is None:
              return labels
         else:
              __UpperCamelCase   							=labels['input_ids']
              return input_features






    def _a (      self ,       *A_ ,       **A_   )  ->	Optional[int]:
         return self.tokenizer.decode(*A_ ,       **A_   )






    @contextmanager
    def _a (      self   )  ->	str:
         warnings.warn(
             '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
             'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
             'your audio inputs, or in a separate call.'   )
         __UpperCamelCase   							=True
         __UpperCamelCase   							=self.tokenizer
         yield
         __UpperCamelCase   							=self.feature_extractor
         __UpperCamelCase   							=False





