import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase__     						=			datasets.utils.logging.get_logger(__name__)
@dataclass
class     a_			(						datasets.BuilderConfig						):






 '''simple docstring'''






 UpperCAmelCase_     			= None
 UpperCAmelCase_     			= "utf-8"
 UpperCAmelCase_     			= None
 UpperCAmelCase_     			= None
 UpperCAmelCase_     			= True  # deprecated
 UpperCAmelCase_     			= None  # deprecated
 UpperCAmelCase_     			= 10 << 20  # 10MB
 UpperCAmelCase_     			= None






class     a_			(						datasets.ArrowBasedBuilder						):






 '''simple docstring'''






 UpperCAmelCase_     			= JsonConfig




 def 		__snake_case						(      self :  Optional[int]):







        '''simple docstring'''



        if self.config.block_size is not None:
               logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead')
               lowerCAmelCase__					     = self.config.block_size
        if self.config.use_threads is not True:
               logger.warning(
                   'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.')
        if self.config.newlines_in_values is not None:
               raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported')
        return datasets.DatasetInfo(features=self.config.features)




 def 		__snake_case						(      self :  Any      ,   lowercase__ :  Any):







        '''simple docstring'''



        if not self.config.data_files:
               raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}""")
        lowerCAmelCase__					     = dl_manager.download_and_extract(self.config.data_files)
        if isinstance(__snake_case      ,   (str, list, tuple)):
               lowerCAmelCase__					     = data_files
               if isinstance(__snake_case      ,   __snake_case):
                      lowerCAmelCase__					     = [files]
               lowerCAmelCase__					     = [dl_manager.iter_files(__snake_case) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,   gen_kwargs={'files': files})]
        lowerCAmelCase__					     = []
        for split_name, files in data_files.items():
               if isinstance(__snake_case      ,   __snake_case):
                      lowerCAmelCase__					     = [files]
               lowerCAmelCase__					     = [dl_manager.iter_files(__snake_case) for file in files]
               splits.append(datasets.SplitGenerator(name=__snake_case      ,   gen_kwargs={'files': files}))
        return splits




 def 		__snake_case						(      self :  Optional[int]      ,   lowercase__ :  pa.Table):







        '''simple docstring'''



        if self.config.features is not None:
               # adding missing columns
               for column_name in set(self.config.features) - set(pa_table.column_names):
                      lowerCAmelCase__					     = self.config.features.arrow_schema.field(__snake_case).type
                      lowerCAmelCase__					     = pa_table.append_column(__snake_case      ,   pa.array([None] * len(__snake_case)      ,   type=__snake_case))
               # more expensive cast to support nested structures with keys in a different order
               # allows str <-> int/float or str to Audio for example
               lowerCAmelCase__					     = table_cast(__snake_case      ,   self.config.features.arrow_schema)
        return pa_table




 def 		__snake_case						(      self :  int      ,   lowercase__ :  Optional[Any]):







        '''simple docstring'''



        for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case)):
               # If the file is one json object and if we need to look at the list of items in one specific field
               if self.config.field is not None:
                      with open(__snake_case      ,   encoding=self.config.encoding      ,   errors=self.config.encoding_errors) as f:
                             lowerCAmelCase__					     = json.load(__snake_case)

                      # We keep only the field we are interested in
                      lowerCAmelCase__					     = dataset[self.config.field]

                      # We accept two format: a list of dicts or a dict of lists
                      if isinstance(__snake_case      ,   (list, tuple)):
                             lowerCAmelCase__					     = set().union(*[row.keys() for row in dataset])
                             lowerCAmelCase__					     = {col: [row.get(__snake_case) for row in dataset] for col in keys}
                      else:
                             lowerCAmelCase__					     = dataset
                      lowerCAmelCase__					     = pa.Table.from_pydict(__snake_case)
                      yield file_idx, self._cast_table(__snake_case)

               # If the file has one json object per line
               else:
                      with open(__snake_case      ,   'rb') as f:
                             lowerCAmelCase__					     = 0
                             # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                             # Set a default minimum value of 16kB if the chunk size is really small
                             lowerCAmelCase__					     = max(self.config.chunksize // 32      ,   16 << 10)
                             lowerCAmelCase__					     = (
                                 self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
                             )
                             while True:
                                    lowerCAmelCase__					     = f.read(self.config.chunksize)
                                    if not batch:
                                           break
                                    # Finish current line
                                    try:
                                           batch += f.readline()
                                    except (AttributeError, io.UnsupportedOperation):
                                           batch += readline(__snake_case)
                                    # PyArrow only accepts utf-8 encoded bytes
                                    if self.config.encoding != "utf-8":
                                           lowerCAmelCase__					     = batch.decode(self.config.encoding      ,   errors=__snake_case).encode('utf-8')
                                    try:
                                           while True:
                                                  try:
                                                         lowerCAmelCase__					     = paj.read_json(
                                                             io.BytesIO(__snake_case)      ,   read_options=paj.ReadOptions(block_size=__snake_case))
                                                         break
                                                  except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                         if (
                                                             isinstance(__snake_case      ,   pa.ArrowInvalid)
                                                             and "straddling" not in str(__snake_case)
                                                             or block_size > len(__snake_case)
                                                         ):
                                                                raise
                                                         else:
                                                                # Increase the block size in case it was too small.
                                                                # The block size will be reset for the next file.
                                                                logger.debug(
                                                                    F"""Batch of {len(__snake_case)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.""")
                                                                block_size *= 2
                                    except pa.ArrowInvalid as e:
                                           try:
                                                  with open(
                                                      __snake_case      ,   encoding=self.config.encoding      ,   errors=self.config.encoding_errors) as f:
                                                         lowerCAmelCase__					     = json.load(__snake_case)
                                           except json.JSONDecodeError:
                                                  logger.error(F"""Failed to read file \'{file}\' with error {type(__snake_case)}: {e}""")
                                                  raise e
                                           # If possible, parse the file as a list of json objects and exit the loop
                                           if isinstance(__snake_case      ,   __snake_case):  # list is the only sequence type supported in JSON
                                                  try:
                                                         lowerCAmelCase__					     = set().union(*[row.keys() for row in dataset])
                                                         lowerCAmelCase__					     = {col: [row.get(__snake_case) for row in dataset] for col in keys}
                                                         lowerCAmelCase__					     = pa.Table.from_pydict(__snake_case)
                                                  except (pa.ArrowInvalid, AttributeError) as e:
                                                         logger.error(F"""Failed to read file \'{file}\' with error {type(__snake_case)}: {e}""")
                                                         raise ValueError(F"""Not able to read records in the JSON file at {file}.""") from None
                                                  yield file_idx, self._cast_table(__snake_case)
                                                  break
                                           else:
                                                  logger.error(F"""Failed to read file \'{file}\' with error {type(__snake_case)}: {e}""")
                                                  raise ValueError(
                                                      F"""Not able to read records in the JSON file at {file}. """
                                                      F"""You should probably indicate the field of the JSON file containing your records. """
                                                      F"""This JSON file contain the following fields: {str(list(dataset.keys()))}. """
                                                      F"""Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. """) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(__snake_case)
                                    batch_idx += 1

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase__     						=			logging.get_logger(__name__)

lowerCAmelCase__     						=			{
    'facebook/xlm-roberta-xl': 'https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json',
    'facebook/xlm-roberta-xxl': 'https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json',
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}
class     a_			(						SCREAMING_SNAKE_CASE						):






 '''simple docstring'''






 UpperCAmelCase_     			= 'xlm-roberta-xl'




 def __init__(      self :  int      ,   lowercase__ :  Optional[int]=250_880      ,   lowercase__ :  Union[str, Any]=2_560      ,   lowercase__ :  int=36      ,   lowercase__ :  int=32      ,   lowercase__ :  List[Any]=10_240      ,   lowercase__ :  Union[str, Any]="gelu"      ,   lowercase__ :  Dict=0.1      ,   lowercase__ :  Any=0.1      ,   lowercase__ :  Any=514      ,   lowercase__ :  Optional[int]=1      ,   lowercase__ :  Any=0.02      ,   lowercase__ :  List[str]=1e-05      ,   lowercase__ :  Dict=1      ,   lowercase__ :  Union[str, Any]=0      ,   lowercase__ :  Dict=2      ,   lowercase__ :  str="absolute"      ,   lowercase__ :  Dict=True      ,   lowercase__ :  List[Any]=None      ,   **lowercase__ :  Optional[Any]      ,   ):







        '''simple docstring'''



        super().__init__(pad_token_id=lowercase__      ,   bos_token_id=lowercase__      ,   eos_token_id=lowercase__      ,   **lowercase__)
        lowerCAmelCase__					     = vocab_size
        lowerCAmelCase__					     = hidden_size
        lowerCAmelCase__					     = num_hidden_layers
        lowerCAmelCase__					     = num_attention_heads
        lowerCAmelCase__					     = hidden_act
        lowerCAmelCase__					     = intermediate_size
        lowerCAmelCase__					     = hidden_dropout_prob
        lowerCAmelCase__					     = attention_probs_dropout_prob
        lowerCAmelCase__					     = max_position_embeddings
        lowerCAmelCase__					     = type_vocab_size
        lowerCAmelCase__					     = initializer_range
        lowerCAmelCase__					     = layer_norm_eps
        lowerCAmelCase__					     = position_embedding_type
        lowerCAmelCase__					     = use_cache
        lowerCAmelCase__					     = classifier_dropout






class     a_			(						SCREAMING_SNAKE_CASE						):






 '''simple docstring'''






 @property
 def 		__snake_case						(      self :  int):







        '''simple docstring'''



        if self.task == "multiple-choice":
               lowerCAmelCase__					     = {0: 'batch', 1: 'choice', 2: 'sequence'}
        else:
               lowerCAmelCase__					     = {0: 'batch', 1: 'sequence'}
        return OrderedDict(
            [
                ('input_ids', dynamic_axis),
                ('attention_mask', dynamic_axis),
            ])
