

"""simple docstring"""





import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


_a							=							logging.getLogger(__name__)




@dataclass(frozen=SCREAMING_SNAKE_CASE__       )
class  _lowerCAmelCase  :







 """simple docstring"""







 __UpperCAmelCase     : str
 __UpperCAmelCase     : str
 __UpperCAmelCase     : Optional[str]       		=     None
 __UpperCAmelCase     : Optional[str]       		=     None
 __UpperCAmelCase     : Optional[str]       		=     None







@dataclass(frozen=SCREAMING_SNAKE_CASE__       )
class  _lowerCAmelCase  :







 """simple docstring"""







 __UpperCAmelCase     : List[int]
 __UpperCAmelCase     : Optional[List[int]]       		=     None
 __UpperCAmelCase     : Optional[List[int]]       		=     None
 __UpperCAmelCase     : Optional[Union[int, float]]       		=     None
 __UpperCAmelCase     : Optional[int]       		=     None


if is_torch_available():
   import torch
   from torch.utils.data import Dataset




   class  _lowerCAmelCase  (     SCREAMING_SNAKE_CASE__       ):







    """simple docstring"""







    __UpperCAmelCase     : List[InputFeatures]


    def __init__(      self							:      str, UpperCAmelCase__							:      str, UpperCAmelCase__							:      PreTrainedTokenizer, UpperCAmelCase__							:      str, UpperCAmelCase__							:      Optional[int] = None, UpperCAmelCase__							:      Optional[Any]=False, UpperCAmelCase__							:      bool = False, ):
       __lowercase								=						hans_processors[task]()

       __lowercase								=						os.path.join(
           a_, "cached_{}_{}_{}_{}".format(
               "dev" if evaluate else "train", tokenizer.__class__.__name__, str(a_     ), a_, ), )
       __lowercase								=						processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
          # HACK(label indices are swapped in RoBERTa pretrained model)
          __lowercase		,__lowercase								=						label_list[2], label_list[1]
       __lowercase								=						label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       __lowercase								=						cached_features_file + ".lock"
       with FileLock(a_     ):
          if os.path.exists(a_     ) and not overwrite_cache:
             logger.info(F"""Loading features from cached file {cached_features_file}"""     )
             __lowercase								=						torch.load(a_     )
          else:
             logger.info(F"""Creating features from dataset file at {data_dir}"""     )

             __lowercase								=						(
                 processor.get_dev_examples(a_     ) if evaluate else processor.get_train_examples(a_     )
             )

             logger.info("Training examples: %s", len(a_     )     )
             __lowercase								=						hans_convert_examples_to_features(a_, a_, a_, a_     )
             logger.info("Saving features into cached file %s", a_     )
             torch.save(self.features, a_     )


    def __len__(      self							:      List[Any]     ):
       return len(self.features     )


    def __getitem__(      self							:      Dict, UpperCAmelCase__							:      List[Any]     ):
       return self.features[i]



    def  _lowercase			(      self							:      Optional[int]     ):
       return self.label_list


if is_tf_available():
   import tensorflow as tf




   class  _lowerCAmelCase  :







    """simple docstring"""







    __UpperCAmelCase     : List[InputFeatures]


    def __init__(      self							:      str, UpperCAmelCase__							:      str, UpperCAmelCase__							:      PreTrainedTokenizer, UpperCAmelCase__							:      str, UpperCAmelCase__							:      Optional[int] = 1_2_8, UpperCAmelCase__							:      Union[str, Any]=False, UpperCAmelCase__							:      bool = False, ):
       __lowercase								=						hans_processors[task]()
       __lowercase								=						processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
          # HACK(label indices are swapped in RoBERTa pretrained model)
          __lowercase		,__lowercase								=						label_list[2], label_list[1]
       __lowercase								=						label_list

       __lowercase								=						processor.get_dev_examples(a_     ) if evaluate else processor.get_train_examples(a_     )
       __lowercase								=						hans_convert_examples_to_features(a_, a_, a_, a_     )

       def gen():
          for ex_index, ex in tqdm.tqdm(enumerate(self.features     ), desc="convert examples to features"     ):
             if ex_index % 1_0_0_0_0 == 0:
                logger.info("Writing example %d of %d" % (ex_index, len(a_     ))     )

             yield (
                 {
                     "example_id": 0,
                     "input_ids": ex.input_ids,
                     "attention_mask": ex.attention_mask,
                     "token_type_ids": ex.token_type_ids,
                 },
                 ex.label,
             )

       __lowercase								=						tf.data.Dataset.from_generator(
           a_, (
               {
                   "example_id": tf.intaa,
                   "input_ids": tf.intaa,
                   "attention_mask": tf.intaa,
                   "token_type_ids": tf.intaa,
               },
               tf.intaa,
           ), (
               {
                   "example_id": tf.TensorShape([]     ),
                   "input_ids": tf.TensorShape([None, None]     ),
                   "attention_mask": tf.TensorShape([None, None]     ),
                   "token_type_ids": tf.TensorShape([None, None]     ),
               },
               tf.TensorShape([]     ),
           ), )


    def  _lowercase			(      self							:      Union[str, Any]     ):
       return self.dataset


    def __len__(      self							:      List[str]     ):
       return len(self.features     )


    def __getitem__(      self							:      Union[str, Any], UpperCAmelCase__							:      int     ):
       return self.features[i]



    def  _lowercase			(      self							:      int     ):
       return self.label_list




class  _lowerCAmelCase  (     SCREAMING_SNAKE_CASE__       ):







 """simple docstring"""







 def  _lowercase			(      self							:      Any, UpperCAmelCase__							:      Dict     ):
    return self._create_examples(self._read_tsv(os.path.join(a_, "heuristics_train_set.txt"     )     ), "train"     )


 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      Tuple     ):
    return self._create_examples(self._read_tsv(os.path.join(a_, "heuristics_evaluation_set.txt"     )     ), "dev"     )


 def  _lowercase			(      self							:      Optional[int]     ):
    return ["contradiction", "entailment", "neutral"]



 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      Optional[int], UpperCAmelCase__							:      List[str]     ):
    __lowercase								=						[]
    for i, line in enumerate(a_     ):
       if i == 0:
          continue
       __lowercase								=						"%s-%s" % (set_type, line[0])
       __lowercase								=						line[5]
       __lowercase								=						line[6]
       __lowercase								=						line[7][2:] if line[7].startswith("ex"     ) else line[7]
       __lowercase								=						line[0]
       examples.append(InputExample(guid=a_, text_a=a_, text_b=a_, label=a_, pairID=a_     )     )
    return examples







def   _A  (  UpperCamelCase_					:	List[InputExample],      UpperCamelCase_					:	List[str],      UpperCamelCase_					:	int,      UpperCamelCase_					:	PreTrainedTokenizer,      )     ->						Optional[Any]:
   '''simple docstring'''


   __lowercase								=						{label: i for i, label in enumerate(_UpperCamelCase)}

   __lowercase								=						[]
   for ex_index, example in tqdm.tqdm(enumerate(_UpperCamelCase),      desc="convert examples to features"):
      if ex_index % 10000 == 0:
         logger.info("Writing example %d" % (ex_index))

      __lowercase								=						tokenizer(
          example.text_a,      example.text_b,      add_special_tokens=_UpperCamelCase,      max_length=_UpperCamelCase,      padding="max_length",      truncation=_UpperCamelCase,      return_overflowing_tokens=_UpperCamelCase,      )

      __lowercase								=						label_map[example.label] if example.label in label_map else 0

      __lowercase								=						int(example.pairID)

      features.append(InputFeatures(**_UpperCamelCase,      label=_UpperCamelCase,      pairID=_UpperCamelCase))

   for i, example in enumerate(examples[:5]):
      logger.info("*** Example ***")
      logger.info(F"""guid: {example}""")
      logger.info(F"""features: {features[i]}""")

   return features


_a							=							{
    "hans": 3,
}







_a							=							{
    "hans": HansProcessor,
}







"""simple docstring"""





import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput




class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __init__(      self							:      List[str], UpperCAmelCase__							:      Union[str, Any], UpperCAmelCase__							:      Any, UpperCAmelCase__							:      Dict     ):
    __lowercase								=						dataset
    __lowercase								=						process
    __lowercase								=						params


 def __len__(      self							:      str     ):
    return len(self.dataset     )



 def __getitem__(      self							:      List[Any], UpperCAmelCase__							:      int     ):
    __lowercase								=						self.dataset[i]
    __lowercase								=						self.process(UpperCAmelCase__, **self.params     )
    return processed







class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __init__(      self							:      Optional[Any], UpperCAmelCase__							:      Dict, UpperCAmelCase__							:      Any, UpperCAmelCase__							:      Union[str, Any], UpperCAmelCase__							:      List[Any]=None     ):
    __lowercase								=						loader
    __lowercase								=						infer
    __lowercase								=						params
    if loader_batch_size == 1:
       # Let's spare some time by deactivating altogether
       __lowercase								=						None
    __lowercase								=						loader_batch_size

    # Internal bookkeeping
    __lowercase								=						None
    __lowercase								=						None


 def __len__(      self							:      str     ):
    return len(self.loader     )


 def __iter__(      self							:      List[str]     ):
    __lowercase								=						iter(self.loader     )
    return self


 def  _lowercase			(      self							:      Union[str, Any]     ):
    if isinstance(self._loader_batch_data, torch.Tensor     ):
       # Batch data is simple tensor, just fetch the slice
       __lowercase								=						self._loader_batch_data[self._loader_batch_index]
    else:
       # Batch data is assumed to be BaseModelOutput (or dict)
       __lowercase								=						{}
       for k, element in self._loader_batch_data.items():
          if isinstance(UpperCAmelCase__, UpperCAmelCase__     ):
             # Convert ModelOutput to tuple first
             __lowercase								=						element.to_tuple()
             if isinstance(element[0], torch.Tensor     ):
                __lowercase								=						tuple(el[self._loader_batch_index].unsqueeze(0     ) for el in element     )
             elif isinstance(element[0], np.ndarray     ):
                __lowercase								=						tuple(np.expand_dims(el[self._loader_batch_index], 0     ) for el in element     )
             continue
          if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(UpperCAmelCase__, UpperCAmelCase__     ):
             # Those are stored as lists of tensors so need specific unbatching.
             if isinstance(element[0], torch.Tensor     ):
                __lowercase								=						tuple(el[self._loader_batch_index].unsqueeze(0     ) for el in element     )
             elif isinstance(element[0], np.ndarray     ):
                __lowercase								=						tuple(np.expand_dims(el[self._loader_batch_index], 0     ) for el in element     )
             continue
          if element is None:
             # This can happen for optional data that get passed around
             __lowercase								=						None
          elif isinstance(element[self._loader_batch_index], torch.Tensor     ):
             # Take correct batch data, but make it looked like batch_size=1
             # For compatibility with other methods within transformers

             __lowercase								=						element[self._loader_batch_index].unsqueeze(0     )
          elif isinstance(element[self._loader_batch_index], np.ndarray     ):
             # Take correct batch data, but make it looked like batch_size=1
             # For compatibility with other methods within transformers
             __lowercase								=						np.expand_dims(element[self._loader_batch_index], 0     )
          else:
             # This is typically a list, so no need to `unsqueeze`.
             __lowercase								=						element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
       __lowercase								=						self._loader_batch_data.__class__(UpperCAmelCase__     )
    self._loader_batch_index += 1
    return result



 def  _lowercase			(      self							:      Tuple     ):
    if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
       # We are currently unrolling a batch so we just need to return
       # the current item within a batch
       return self.loader_batch_item()

    # We're out of items within a batch
    __lowercase								=						next(self.iterator     )
    __lowercase								=						self.infer(UpperCAmelCase__, **self.params     )
    # We now have a batch of "inferred things".
    if self.loader_batch_size is not None:
       # Try to infer the size of the batch
       if isinstance(UpperCAmelCase__, torch.Tensor     ):
          __lowercase								=						processed
       else:
          __lowercase								=						list(processed.keys()     )[0]
          __lowercase								=						processed[key]
       if isinstance(UpperCAmelCase__, UpperCAmelCase__     ):
          __lowercase								=						len(UpperCAmelCase__     )
       else:
          __lowercase								=						first_tensor.shape[0]
       if 0 < observed_batch_size < self.loader_batch_size:
          # could be last batch so we can't unroll as many
          # elements.
          __lowercase								=						observed_batch_size
       # Setting internal index to unwrap the batch
       __lowercase								=						processed
       __lowercase								=						0
       return self.loader_batch_item()
    else:
       # We're not unrolling batches
       return processed







class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __init__(      self							:      Union[str, Any], UpperCAmelCase__							:      Any, UpperCAmelCase__							:      List[str], UpperCAmelCase__							:      int, UpperCAmelCase__							:      str=None     ):
    super().__init__(UpperCAmelCase__, UpperCAmelCase__, UpperCAmelCase__     )


 def __iter__(      self							:      str     ):
    __lowercase								=						iter(self.loader     )
    __lowercase								=						None
    return self



 def  _lowercase			(      self							:      int     ):
    if self.subiterator is None:
       __lowercase								=						self.infer(next(self.iterator     ), **self.params     )
    try:
       # Try to return next item
       __lowercase								=						next(self.subiterator     )
    except StopIteration:
       # When a preprocess iterator ends, we can start lookig at the next item
       # ChunkIterator will keep feeding until ALL elements of iterator
       # all have created their subiterator and have been iterating against.
       #
       # Another way to look at it, is we're basically flattening lists of lists
       # into a single list, but with generators
       __lowercase								=						self.infer(next(self.iterator     ), **self.params     )
       __lowercase								=						next(self.subiterator     )
    return processed







class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __iter__(      self							:      int     ):
    __lowercase								=						iter(self.loader     )
    return self



 def  _lowercase			(      self							:      List[str]     ):
    # Extremely similar to PipelineIterator in its unpacking mechanism
    # BUT, we have an extra required item which is the presence of `is_last`
    # That is because everything is flattened by `PipelineChunkIterator` we
    # need to keep track of how to regroup here in the original `process`
    # boundaries so that `process` and `postprocess` see the same data.

    # This iterator accumulates items (possibly while unbatching) until it
    # its a `is_last` and then just passes it on to the caller.
    __lowercase								=						False
    __lowercase								=						[]
    if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
       while self._loader_batch_index < self.loader_batch_size:
          __lowercase								=						self.loader_batch_item()
          __lowercase								=						item.pop("is_last"     )
          accumulator.append(UpperCAmelCase__     )
          if is_last:
             return accumulator

    while not is_last:
       __lowercase								=						self.infer(next(self.iterator     ), **self.params     )
       if self.loader_batch_size is not None:
          if isinstance(UpperCAmelCase__, torch.Tensor     ):
             __lowercase								=						processed
          else:
             __lowercase								=						list(processed.keys()     )[0]
             __lowercase								=						processed[key]
          if isinstance(UpperCAmelCase__, UpperCAmelCase__     ):
             __lowercase								=						len(UpperCAmelCase__     )
          else:
             __lowercase								=						first_tensor.shape[0]
          if 0 < observed_batch_size < self.loader_batch_size:
             # could be last batch so we can't unroll as many
             # elements.
             __lowercase								=						observed_batch_size
          __lowercase								=						processed
          __lowercase								=						0
          while self._loader_batch_index < self.loader_batch_size:
             __lowercase								=						self.loader_batch_item()
             __lowercase								=						item.pop("is_last"     )
             accumulator.append(UpperCAmelCase__     )
             if is_last:
                return accumulator
       else:
          __lowercase								=						processed
          __lowercase								=						item.pop("is_last"     )
          accumulator.append(UpperCAmelCase__     )
    return accumulator







class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __init__(      self							:      List[Any], UpperCAmelCase__							:      Dataset, UpperCAmelCase__							:      str     ):
    __lowercase								=						dataset
    __lowercase								=						key


 def __len__(      self							:      Optional[Any]     ):
    return len(self.dataset     )



 def __getitem__(      self							:      Union[str, Any], UpperCAmelCase__							:      Any     ):
    return self.dataset[i][self.key]







class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 def __init__(      self							:      str, UpperCAmelCase__							:      Dataset, UpperCAmelCase__							:      str, UpperCAmelCase__							:      str     ):
    __lowercase								=						dataset
    __lowercase								=						keya
    __lowercase								=						keya


 def __len__(      self							:      Optional[int]     ):
    return len(self.dataset     )



 def __getitem__(      self							:      Dict, UpperCAmelCase__							:      Tuple     ):
    return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}




