from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


UpperCamelCase__    							= logging.get_logger(__name__)

# General docstring
UpperCamelCase__    							= 'RegNetConfig'

# Base docstring
UpperCamelCase__    							= 'facebook/regnet-y-040'
UpperCamelCase__    							= [1, 1_0_8_8, 7, 7]

# Image classification docstring
UpperCamelCase__    							= 'facebook/regnet-y-040'
UpperCamelCase__    							= 'tabby, tabby cat'

UpperCamelCase__    							= [
    'facebook/regnet-y-040',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]





class 					A							(    nn.Module					):


   def __init__(self					:						Optional[Any]	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int = 3	,  __UpperCAmelCase					:						int = 1	,  __UpperCAmelCase					:						int = 1	,  __UpperCAmelCase					:						Optional[str] = "relu"	,  )						->    Any:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  nn.Convad(
             __UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=__UpperCAmelCase	,  stride=__UpperCAmelCase	,  padding=kernel_size // 2	,  groups=__UpperCAmelCase	,  bias=__UpperCAmelCase	,  )
         UpperCAmelCase__								=  nn.BatchNormad(__UpperCAmelCase		)
         UpperCAmelCase__								=  ACTaFN[activation] if activation is not None else nn.Identity()





   def       lowercase_				(self					:						str	,  __UpperCAmelCase					:						int		)						->    Tuple:
         """simple docstring"""



         UpperCAmelCase__								=  self.convolution(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.normalization(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.activation(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						Dict	,  __UpperCAmelCase					:						RegNetConfig		)						->    Optional[int]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  RegNetConvLayer(
             config.num_channels	,  config.embedding_size	,  kernel_size=3	,  stride=2	,  activation=config.hidden_act		)
         UpperCAmelCase__								=  config.num_channels





   def       lowercase_				(self					:						List[Any]	,  __UpperCAmelCase					:						Optional[Any]		)						->    str:
         """simple docstring"""



         UpperCAmelCase__								=  pixel_values.shape[1]
         if num_channels != self.num_channels:
               raise ValueError(
                   "Make sure that the channel dimension of the pixel values match with the one set in the configuration."		)
         UpperCAmelCase__								=  self.embedder(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						Optional[int]	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int = 2		)						->    List[str]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  nn.Convad(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1	,  stride=__UpperCAmelCase	,  bias=__UpperCAmelCase		)
         UpperCAmelCase__								=  nn.BatchNormad(__UpperCAmelCase		)





   def       lowercase_				(self					:						Optional[int]	,  __UpperCAmelCase					:						Tensor		)						->    Tensor:
         """simple docstring"""



         UpperCAmelCase__								=  self.convolution(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.normalization(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						Optional[int]	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int		)						->    Tuple:
         """simple docstring"""



         super().__init__()

         UpperCAmelCase__								=  nn.AdaptiveAvgPoolad((1, 1)		)
         UpperCAmelCase__								=  nn.Sequential(
             nn.Convad(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1		)	,  nn.ReLU()	,  nn.Convad(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1		)	,  nn.Sigmoid()	,  )





   def       lowercase_				(self					:						int	,  __UpperCAmelCase					:						Optional[int]		)						->    str:
         """simple docstring"""



         UpperCAmelCase__								=  self.pooler(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.attention(__UpperCAmelCase		)
         UpperCAmelCase__								=  hidden_state * attention
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						str	,  __UpperCAmelCase					:						RegNetConfig	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int = 1		)						->    Optional[Any]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  in_channels != out_channels or stride != 1
         UpperCAmelCase__								=  max(1	,  out_channels // config.groups_width		)
         UpperCAmelCase__								=  (
             RegNetShortCut(__UpperCAmelCase	,  __UpperCAmelCase	,  stride=__UpperCAmelCase		) if should_apply_shortcut else nn.Identity()
         )
         UpperCAmelCase__								=  nn.Sequential(
             RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1	,  activation=config.hidden_act		)	,  RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  stride=__UpperCAmelCase	,  groups=__UpperCAmelCase	,  activation=config.hidden_act		)	,  RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1	,  activation=__UpperCAmelCase		)	,  )
         UpperCAmelCase__								=  ACTaFN[config.hidden_act]





   def       lowercase_				(self					:						Optional[int]	,  __UpperCAmelCase					:						Optional[int]		)						->    int:
         """simple docstring"""



         UpperCAmelCase__								=  hidden_state
         UpperCAmelCase__								=  self.layer(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.shortcut(__UpperCAmelCase		)
         hidden_state += residual
         UpperCAmelCase__								=  self.activation(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						str	,  __UpperCAmelCase					:						RegNetConfig	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int = 1		)						->    Optional[int]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  in_channels != out_channels or stride != 1
         UpperCAmelCase__								=  max(1	,  out_channels // config.groups_width		)
         UpperCAmelCase__								=  (
             RegNetShortCut(__UpperCAmelCase	,  __UpperCAmelCase	,  stride=__UpperCAmelCase		) if should_apply_shortcut else nn.Identity()
         )
         UpperCAmelCase__								=  nn.Sequential(
             RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1	,  activation=config.hidden_act		)	,  RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  stride=__UpperCAmelCase	,  groups=__UpperCAmelCase	,  activation=config.hidden_act		)	,  RegNetSELayer(__UpperCAmelCase	,  reduced_channels=int(round(in_channels / 4		)		)		)	,  RegNetConvLayer(__UpperCAmelCase	,  __UpperCAmelCase	,  kernel_size=1	,  activation=__UpperCAmelCase		)	,  )
         UpperCAmelCase__								=  ACTaFN[config.hidden_act]





   def       lowercase_				(self					:						List[str]	,  __UpperCAmelCase					:						int		)						->    str:
         """simple docstring"""



         UpperCAmelCase__								=  hidden_state
         UpperCAmelCase__								=  self.layer(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.shortcut(__UpperCAmelCase		)
         hidden_state += residual
         UpperCAmelCase__								=  self.activation(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						Tuple	,  __UpperCAmelCase					:						RegNetConfig	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						int = 2	,  __UpperCAmelCase					:						int = 2	,  )						->    Any:
         """simple docstring"""



         super().__init__()

         UpperCAmelCase__								=  RegNetXLayer if config.layer_type == "x" else RegNetYLayer

         UpperCAmelCase__								=  nn.Sequential(
             # downsampling is done in the first layer with stride of 2
             layer(
                 __UpperCAmelCase	,  __UpperCAmelCase	,  __UpperCAmelCase	,  stride=__UpperCAmelCase	,  )	,  *[layer(__UpperCAmelCase	,  __UpperCAmelCase	,  __UpperCAmelCase		) for _ in range(depth - 1		)]	,  )





   def       lowercase_				(self					:						Tuple	,  __UpperCAmelCase					:						Optional[int]		)						->    Tuple:
         """simple docstring"""



         UpperCAmelCase__								=  self.layers(__UpperCAmelCase		)
         return hidden_state





class 					A							(    nn.Module					):


   def __init__(self					:						str	,  __UpperCAmelCase					:						RegNetConfig		)						->    Tuple:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=  nn.ModuleList([]		)
         # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
         self.stages.append(
             RegNetStage(
                 __UpperCAmelCase	,  config.embedding_size	,  config.hidden_sizes[0]	,  stride=2 if config.downsample_in_first_stage else 1	,  depth=config.depths[0]	,  )		)
         UpperCAmelCase__								=  zip(config.hidden_sizes	,  config.hidden_sizes[1:]		)
         for (in_channels, out_channels), depth in zip(__UpperCAmelCase	,  config.depths[1:]		):
               self.stages.append(RegNetStage(__UpperCAmelCase	,  __UpperCAmelCase	,  __UpperCAmelCase	,  depth=__UpperCAmelCase		)		)





   def       lowercase_				(self					:						List[Any]	,  __UpperCAmelCase					:						Tensor	,  __UpperCAmelCase					:						bool = False	,  __UpperCAmelCase					:						bool = True		)						->    BaseModelOutputWithNoAttention:
         """simple docstring"""



         UpperCAmelCase__								=  () if output_hidden_states else None

         for stage_module in self.stages:
               if output_hidden_states:
                     UpperCAmelCase__								=  hidden_states + (hidden_state,)

               UpperCAmelCase__								=  stage_module(__UpperCAmelCase		)

         if output_hidden_states:
               UpperCAmelCase__								=  hidden_states + (hidden_state,)

         if not return_dict:
               return tuple(v for v in [hidden_state, hidden_states] if v is not None		)

         return BaseModelOutputWithNoAttention(last_hidden_state=__UpperCAmelCase	,  hidden_states=__UpperCAmelCase		)





class 					A							(    UpperCAmelCase_					):
   __UpperCAmelCase			:			Union[str, Any]              =      RegNetConfig
   __UpperCAmelCase			:			Dict              =      'regnet'
   __UpperCAmelCase			:			List[Any]              =      'pixel_values'
   __UpperCAmelCase			:			Tuple              =      True


   def       lowercase_				(self					:						Dict	,  __UpperCAmelCase					:						List[str]		)						->    Optional[Any]:
         """simple docstring"""



         if isinstance(__UpperCAmelCase	,  nn.Convad		):
               nn.init.kaiming_normal_(module.weight	,  mode="fan_out"	,  nonlinearity="relu"		)
         elif isinstance(__UpperCAmelCase	,  (nn.BatchNormad, nn.GroupNorm)		):
               nn.init.constant_(module.weight	,  1		)
               nn.init.constant_(module.bias	,  0		)





   def       lowercase_				(self					:						Dict	,  __UpperCAmelCase					:						int	,  __UpperCAmelCase					:						str=False		)						->    int:
         """simple docstring"""



         if isinstance(__UpperCAmelCase	,  __UpperCAmelCase		):
               UpperCAmelCase__								=  value




UpperCamelCase__    							= R'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

UpperCamelCase__    							= R'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.\n'





@add_start_docstrings(
    'The bare RegNet model outputting raw features without any specific head on top.'		,    UpperCAmelCase_		,    )
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class 					A							(    UpperCAmelCase_					):


   def __init__(self					:						Union[str, Any]	,  __UpperCAmelCase					:						List[str]		)						->    List[str]:
         """simple docstring"""



         super().__init__(__UpperCAmelCase		)
         UpperCAmelCase__								=  config
         UpperCAmelCase__								=  RegNetEmbeddings(__UpperCAmelCase		)
         UpperCAmelCase__								=  RegNetEncoder(__UpperCAmelCase		)
         UpperCAmelCase__								=  nn.AdaptiveAvgPoolad((1, 1)		)
         # Initialize weights and apply final processing
         self.post_init()





   @add_start_docstrings_to_model_forward(__UpperCAmelCase		)
   @add_code_sample_docstrings(
       checkpoint=_CHECKPOINT_FOR_DOC	,  output_type=__UpperCAmelCase	,  config_class=_CONFIG_FOR_DOC	,  modality="vision"	,  expected_output=_EXPECTED_OUTPUT_SHAPE	,  )
   def       lowercase_				(self					:						Dict	,  __UpperCAmelCase					:						Tensor	,  __UpperCAmelCase					:						Optional[bool] = None	,  __UpperCAmelCase					:						Optional[bool] = None		)						->    BaseModelOutputWithPoolingAndNoAttention:
         """simple docstring"""



         UpperCAmelCase__								=  (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
         )
         UpperCAmelCase__								=  return_dict if return_dict is not None else self.config.use_return_dict

         UpperCAmelCase__								=  self.embedder(__UpperCAmelCase		)

         UpperCAmelCase__								=  self.encoder(
             __UpperCAmelCase	,  output_hidden_states=__UpperCAmelCase	,  return_dict=__UpperCAmelCase		)

         UpperCAmelCase__								=  encoder_outputs[0]

         UpperCAmelCase__								=  self.pooler(__UpperCAmelCase		)

         if not return_dict:
               return (last_hidden_state, pooled_output) + encoder_outputs[1:]

         return BaseModelOutputWithPoolingAndNoAttention(
             last_hidden_state=__UpperCAmelCase	,  pooler_output=__UpperCAmelCase	,  hidden_states=encoder_outputs.hidden_states	,  )





@add_start_docstrings(
    '\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '		,    UpperCAmelCase_		,    )
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class 					A							(    UpperCAmelCase_					):


   def __init__(self					:						Union[str, Any]	,  __UpperCAmelCase					:						str		)						->    Tuple:
         """simple docstring"""



         super().__init__(__UpperCAmelCase		)
         UpperCAmelCase__								=  config.num_labels
         UpperCAmelCase__								=  RegNetModel(__UpperCAmelCase		)
         # classification head
         UpperCAmelCase__								=  nn.Sequential(
             nn.Flatten()	,  nn.Linear(config.hidden_sizes[-1]	,  config.num_labels		) if config.num_labels > 0 else nn.Identity()	,  )
         # initialize weights and apply final processing
         self.post_init()





   @add_start_docstrings_to_model_forward(__UpperCAmelCase		)
   @add_code_sample_docstrings(
       checkpoint=_IMAGE_CLASS_CHECKPOINT	,  output_type=__UpperCAmelCase	,  config_class=_CONFIG_FOR_DOC	,  expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT	,  )
   def       lowercase_				(self					:						int	,  __UpperCAmelCase					:						Optional[torch.FloatTensor] = None	,  __UpperCAmelCase					:						Optional[torch.LongTensor] = None	,  __UpperCAmelCase					:						Optional[bool] = None	,  __UpperCAmelCase					:						Optional[bool] = None	,  )						->    ImageClassifierOutputWithNoAttention:
         """simple docstring"""



         UpperCAmelCase__								=  return_dict if return_dict is not None else self.config.use_return_dict

         UpperCAmelCase__								=  self.regnet(__UpperCAmelCase	,  output_hidden_states=__UpperCAmelCase	,  return_dict=__UpperCAmelCase		)

         UpperCAmelCase__								=  outputs.pooler_output if return_dict else outputs[1]

         UpperCAmelCase__								=  self.classifier(__UpperCAmelCase		)

         UpperCAmelCase__								=  None

         if labels is not None:
               if self.config.problem_type is None:
                     if self.num_labels == 1:
                           UpperCAmelCase__								=  "regression"
                     elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                           UpperCAmelCase__								=  "single_label_classification"
                     else:
                           UpperCAmelCase__								=  "multi_label_classification"
               if self.config.problem_type == "regression":
                     UpperCAmelCase__								=  MSELoss()
                     if self.num_labels == 1:
                           UpperCAmelCase__								=  loss_fct(logits.squeeze()	,  labels.squeeze()		)
                     else:
                           UpperCAmelCase__								=  loss_fct(__UpperCAmelCase	,  __UpperCAmelCase		)
               elif self.config.problem_type == "single_label_classification":
                     UpperCAmelCase__								=  CrossEntropyLoss()
                     UpperCAmelCase__								=  loss_fct(logits.view(-1	,  self.num_labels		)	,  labels.view(-1		)		)
               elif self.config.problem_type == "multi_label_classification":
                     UpperCAmelCase__								=  BCEWithLogitsLoss()
                     UpperCAmelCase__								=  loss_fct(__UpperCAmelCase	,  __UpperCAmelCase		)

         if not return_dict:
               UpperCAmelCase__								=  (logits,) + outputs[2:]
               return (loss,) + output if loss is not None else output

         return ImageClassifierOutputWithNoAttention(loss=__UpperCAmelCase	,  logits=__UpperCAmelCase	,  hidden_states=outputs.hidden_states		)




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__    							= logging.getLogger(__name__)




def        lowerCAmelCase_			(					)					->    Union[str, Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  argparse.ArgumentParser(
          description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
      parser.add_argument(
          "--dataset_name",  type=__A,  default="wikitext",  help="Name of the training. Explore datasets at: hf.co/datasets.",  )
      parser.add_argument(
          "--dataset_config",  type=__A,  default="wikitext-103-raw-v1",  help="Configuration name of the dataset."							)
      parser.add_argument(
          "--tokenizer_name_or_path",  type=__A,  default="sayakpaul/unigram-tokenizer-wikitext",  help="Tokenizer identifier. Can be a local filepath or a Hub identifier.",  )
      parser.add_argument(
          "--shard_size",  type=__A,  default=1_000,  help="Number of entries to go in a single shard.",  )
      parser.add_argument("--split",  type=__A,  default="train",  choices=["train", "test", "validation"]							)
      parser.add_argument(
          "--limit",  default=__A,  type=__A,  help="Limit the number of shards (used for debugging).",  )
      parser.add_argument(
          "--max_length",  type=__A,  default=512,  help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
          " sequence length that is a multiple of 8.",  )
      parser.add_argument(
          "--output_dir",  default="tf-tpu",  type=__A,  help="Output directory where the TFRecord shards will be saved. If the"
          " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
          " shards will be directly saved to a Google Cloud Storage bucket.",  )

      UpperCAmelCase__								=  parser.parse_args()
      return args




def        lowerCAmelCase_			(					__A							)					->    Optional[int]:







      '''simple docstring'''


      def fn(__A							):
            return tokenizer(examples["text"]							)

      return fn




def        lowerCAmelCase_			(					__A							)					->    List[Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  []
      for i in range(len(tokenized_data["input_ids"]							)							):
            UpperCAmelCase__								=  {
                "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
                "attention_mask": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
            }
            UpperCAmelCase__								=  tf.train.Features(feature=__A							)
            UpperCAmelCase__								=  tf.train.Example(features=__A							)
            UpperCAmelCase__								=  example.SerializeToString()
            records.append(__A							)
      return records




def        lowerCAmelCase_			(					__A							)					->    Union[str, Any]:







      '''simple docstring'''


      UpperCAmelCase__								=  datasets.load_dataset(args.dataset_name,  args.dataset_config,  split=args.split							)

      if args.limit is not None:
            UpperCAmelCase__								=  min(len(__A							),  args.limit							)
            UpperCAmelCase__								=  dataset.select(range(__A							)							)
            print(f"""Limiting the dataset to {args.limit} entries."""							)

      UpperCAmelCase__								=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir							):
                  os.makedirs(args.output_dir							)
            UpperCAmelCase__								=  os.path.join(args.output_dir,  args.split							)
            if not os.path.exists(__A							):
                  os.makedirs(__A							)
      else:
            UpperCAmelCase__								=  os.path.join(args.output_dir,  args.split							)

      # Tokenize the whole dataset at once.
      UpperCAmelCase__								=  tokenize_function(__A							)
      UpperCAmelCase__								=  dataset.map(__A,  batched=__A,  num_proc=4,  remove_columns=["text"]							)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(__A							):
            # Concatenate all texts.
            UpperCAmelCase__								=  {k: sum(examples[k],  []							) for k in examples.keys()}
            UpperCAmelCase__								=  len(concatenated_examples[list(examples.keys()							)[0]]							)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            UpperCAmelCase__								=  (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            UpperCAmelCase__								=  {
                k: [t[i : i + args.max_length] for i in range(0,  __A,  args.max_length							)]
                for k, t in concatenated_examples.items()
            }
            return result

      UpperCAmelCase__								=  dataset_tokenized.map(__A,  batched=__A,  batch_size=1_000,  num_proc=4							)

      UpperCAmelCase__								=  0
      UpperCAmelCase__								=  0
      for shard in range(0,  len(__A							),  args.shard_size							):
            UpperCAmelCase__								=  grouped_dataset[shard : shard + args.shard_size]
            UpperCAmelCase__								=  len(dataset_snapshot["input_ids"]							)
            UpperCAmelCase__								=  os.path.join(__A,  f"""dataset-{shard_count}-{records_containing}.tfrecord"""							)
            UpperCAmelCase__								=  get_serialized_examples(__A							)

            with tf.io.TFRecordWriter(__A							) as out_file:
                  for i in range(len(__A							)							):
                        UpperCAmelCase__								=  serialized_examples[i]
                        out_file.write(__A							)
                  print("Wrote file {} containing {} records".format(__A,  __A							)							)

            shard_count += 1
            total_records += records_containing

      with open(f"""split-{args.split}-records-count.txt""",  "w"							) as f:
            print(f"""Total {args.split} records: {total_records}""",  file=__A							)


if __name__ == "__main__":
      UpperCamelCase__    							= parse_args()
      main(args)



