






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__magic_name__:					List[str]							=     logging.getLogger(__name__)





@dataclass(frozen=_lowerCAmelCase				)
class 	snake_case__ :
	lowercase__						:					str
	lowercase__						:					str
	lowercase__						:					Optional[str]											=   None
	lowercase__						:					Optional[str]											=   None
	lowercase__						:					Optional[str]											=   None





@dataclass(frozen=_lowerCAmelCase				)
class 	snake_case__ :
	lowercase__						:					List[int]
	lowercase__						:					Optional[List[int]]											=   None
	lowercase__						:					Optional[List[int]]											=   None
	lowercase__						:					Optional[Union[int, float]]											=   None
	lowercase__						:					Optional[int]											=   None


if is_torch_available():
		import torch
		from torch.utils.data import Dataset





		class 	snake_case__ (							_lowerCAmelCase				):
			lowercase__						:					List[InputFeatures]
			def __init__(    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__ = False      ,       )							->				Dict:
									__magic_name__      :       Dict       						= hans_processors[task]()

									__magic_name__      :       Dict       						= os.path.join(
									    lowerCAmelCase__      ,       """cached_{}_{}_{}_{}""".format(
									        """dev""" if evaluate else """train"""      ,       tokenizer.__class__.__name__      ,       str(lowerCAmelCase__			)      ,       lowerCAmelCase__      ,       )      ,       )
									__magic_name__      :       List[str]       						= processor.get_labels()
									if tokenizer.__class__ in (
									    RobertaTokenizer,
									    RobertaTokenizerFast,
									    XLMRobertaTokenizer,
									    BartTokenizer,
									    BartTokenizerFast,
									):
															# HACK(label indices are swapped in RoBERTa pretrained model)
															__magic_name__						,__magic_name__      :       int       						= label_list[2], label_list[1]
									__magic_name__      :       Tuple       						= label_list

									# Make sure only the first process in distributed training processes the dataset,
									# and the others will use the cache.
									__magic_name__      :       Tuple       						= cached_features_file + """.lock"""
									with FileLock(lowerCAmelCase__			):
															if os.path.exists(lowerCAmelCase__			) and not overwrite_cache:
																					logger.info(F'Loading features from cached file {cached_features_file}'			)
																					__magic_name__      :       Any       						= torch.load(lowerCAmelCase__			)
															else:
																					logger.info(F'Creating features from dataset file at {data_dir}'			)

																					__magic_name__      :       List[str]       						= (
																					    processor.get_dev_examples(lowerCAmelCase__			) if evaluate else processor.get_train_examples(lowerCAmelCase__			)
																					)

																					logger.info("""Training examples: %s"""      ,       len(lowerCAmelCase__			)			)
																					__magic_name__      :       Dict       						= hans_convert_examples_to_features(lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)
																					logger.info("""Saving features into cached file %s"""      ,       lowerCAmelCase__			)
																					torch.save(self.features      ,       lowerCAmelCase__			)
			def __len__(    self			)							->				Optional[Any]:
									return len(self.features			)
			def __getitem__(    self      ,       lowerCAmelCase__			)							->				InputFeatures:
									return self.features[i]







			def       __magic_name__    (    self			)							->				Optional[int]:
									return self.label_list


if is_tf_available():
		import tensorflow as tf





		class 	snake_case__ :
			lowercase__						:					List[InputFeatures]
			def __init__(    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = 1_28      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__ = False      ,       )							->				str:
									__magic_name__      :       Optional[Any]       						= hans_processors[task]()
									__magic_name__      :       Dict       						= processor.get_labels()
									if tokenizer.__class__ in (
									    RobertaTokenizer,
									    RobertaTokenizerFast,
									    XLMRobertaTokenizer,
									    BartTokenizer,
									    BartTokenizerFast,
									):
															# HACK(label indices are swapped in RoBERTa pretrained model)
															__magic_name__						,__magic_name__      :       Any       						= label_list[2], label_list[1]
									__magic_name__      :       List[Any]       						= label_list

									__magic_name__      :       str       						= processor.get_dev_examples(lowerCAmelCase__			) if evaluate else processor.get_train_examples(lowerCAmelCase__			)
									__magic_name__      :       Dict       						= hans_convert_examples_to_features(lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)

									def gen():
															for ex_index, ex in tqdm.tqdm(enumerate(self.features			)      ,       desc="""convert examples to features"""			):
																					if ex_index % 1_00_00 == 0:
																											logger.info("""Writing example %d of %d""" % (ex_index, len(lowerCAmelCase__			))			)

																					yield (
																					    {
																					        "example_id": 0,
																					        "input_ids": ex.input_ids,
																					        "attention_mask": ex.attention_mask,
																					        "token_type_ids": ex.token_type_ids,
																					    },
																					    ex.label,
																					)

									__magic_name__      :       Dict       						= tf.data.Dataset.from_generator(
									    lowerCAmelCase__      ,       (
									        {
									            """example_id""": tf.intaa,
									            """input_ids""": tf.intaa,
									            """attention_mask""": tf.intaa,
									            """token_type_ids""": tf.intaa,
									        },
									        tf.intaa,
									    )      ,       (
									        {
									            """example_id""": tf.TensorShape([]			),
									            """input_ids""": tf.TensorShape([None, None]			),
									            """attention_mask""": tf.TensorShape([None, None]			),
									            """token_type_ids""": tf.TensorShape([None, None]			),
									        },
									        tf.TensorShape([]			),
									    )      ,       )
			def       __magic_name__    (    self			)							->				Any:
									return self.dataset
			def __len__(    self			)							->				Dict:
									return len(self.features			)
			def __getitem__(    self      ,       lowerCAmelCase__			)							->				InputFeatures:
									return self.features[i]







			def       __magic_name__    (    self			)							->				Optional[int]:
									return self.label_list





class 	snake_case__ (							_lowerCAmelCase				):
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				Optional[Any]:
							return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase__      ,       """heuristics_train_set.txt"""			)			)      ,       """train"""			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				int:
							return self._create_examples(self._read_tsv(os.path.join(lowerCAmelCase__      ,       """heuristics_evaluation_set.txt"""			)			)      ,       """dev"""			)
	def       __magic_name__    (    self			)							->				List[str]:
							return ["contradiction", "entailment", "neutral"]







	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				str:
							__magic_name__      :       Tuple       						= []
							for i, line in enumerate(lowerCAmelCase__			):
													if i == 0:
																			continue
													__magic_name__      :       List[str]       						= """%s-%s""" % (set_type, line[0])
													__magic_name__      :       Any       						= line[5]
													__magic_name__      :       List[str]       						= line[6]
													__magic_name__      :       List[Any]       						= line[7][2:] if line[7].startswith("""ex"""			) else line[7]
													__magic_name__      :       List[Any]       						= line[0]
													examples.append(InputExample(guid=lowerCAmelCase__      ,       text_a=lowerCAmelCase__      ,       text_b=lowerCAmelCase__      ,       label=lowerCAmelCase__      ,       pairID=lowerCAmelCase__			)			)
							return examples



def    UpperCamelCase						(	_A,   _A,   _A,   _A,   ):






						"""simple docstring"""







						__magic_name__      :       Union[str, Any]       						= {label: i for i, label in enumerate(_A			)}

						__magic_name__      :       Any       						= []
						for ex_index, example in tqdm.tqdm(enumerate(_A			),   desc="""convert examples to features"""			):
												if ex_index % 10000 == 0:
																		logger.info("""Writing example %d""" % (ex_index)			)

												__magic_name__      :       List[Any]       						= tokenizer(
												    example.text_a,   example.text_b,   add_special_tokens=_A,   max_length=_A,   padding="""max_length""",   truncation=_A,   return_overflowing_tokens=_A,   )

												__magic_name__      :       List[str]       						= label_map[example.label] if example.label in label_map else 0

												__magic_name__      :       Any       						= int(example.pairID			)

												features.append(InputFeatures(**_A,   label=_A,   pairID=_A			)			)

						for i, example in enumerate(examples[:5]			):
												logger.info("""*** Example ***"""			)
												logger.info(f'guid: {example}'			)
												logger.info(f'features: {features[i]}'			)

						return features


__magic_name__:					str							=     {
    "hans": 3,
}

__magic_name__:					List[Any]							=     {
    "hans": HansProcessor,
}








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		__magic_name__:					Any							=     None


__magic_name__:					Dict							=     logging.get_logger(__name__)

__magic_name__:					Any							=     {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

__magic_name__:					str							=     {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json",
    },
}

__magic_name__:					Optional[Any]							=     {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

__magic_name__:					Optional[Any]							=     "▁"

# Segments (not really needed)
__magic_name__:					List[Any]							=     0
__magic_name__:					Dict							=     1
__magic_name__:					List[str]							=     2
__magic_name__:					List[Any]							=     3
__magic_name__:					Optional[int]							=     4





class 	snake_case__ (							_lowerCAmelCase				):
	lowercase__						:					Dict											=   VOCAB_FILES_NAMES
	lowercase__						:					Union[str, Any]											=   PRETRAINED_VOCAB_FILES_MAP
	lowercase__						:					Tuple											=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						:					List[str]											=   '''left'''
	lowercase__						:					List[str]											=   XLNetTokenizer
	def __init__(    self      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__=True      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="<unk>"      ,       lowerCAmelCase__="<sep>"      ,       lowerCAmelCase__="<pad>"      ,       lowerCAmelCase__="<cls>"      ,       lowerCAmelCase__="<mask>"      ,       lowerCAmelCase__=["<eop>", "<eod>"]      ,       **lowerCAmelCase__      ,       )							->				Tuple:
							# Mask token behave like a normal word, i.e. include the space before it
							__magic_name__      :       Optional[int]       						= AddedToken(lowerCAmelCase__      ,       lstrip=lowerCAmelCase__      ,       rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			) else mask_token

							super().__init__(
							    vocab_file=lowerCAmelCase__      ,       tokenizer_file=lowerCAmelCase__      ,       do_lower_case=lowerCAmelCase__      ,       remove_space=lowerCAmelCase__      ,       keep_accents=lowerCAmelCase__      ,       bos_token=lowerCAmelCase__      ,       eos_token=lowerCAmelCase__      ,       unk_token=lowerCAmelCase__      ,       sep_token=lowerCAmelCase__      ,       pad_token=lowerCAmelCase__      ,       cls_token=lowerCAmelCase__      ,       mask_token=lowerCAmelCase__      ,       additional_special_tokens=lowerCAmelCase__      ,       **lowerCAmelCase__      ,       )

							__magic_name__      :       List[str]       						= 3
							__magic_name__      :       str       						= do_lower_case
							__magic_name__      :       Union[str, Any]       						= remove_space
							__magic_name__      :       str       						= keep_accents
							__magic_name__      :       Tuple       						= vocab_file
							__magic_name__      :       List[Any]       						= False if not self.vocab_file else True
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       Any       						= [self.sep_token_id]
							__magic_name__      :       Any       						= [self.cls_token_id]
							if token_ids_a is None:
													return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       List[str]       						= [self.sep_token_id]
							__magic_name__      :       Optional[Any]       						= [2]

							if token_ids_a is None:
													return len(token_ids_a + sep			) * [0] + cls_segment_id
							return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id







	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				Tuple[str]:
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""			)

							if not os.path.isdir(lowerCAmelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
													return
							__magic_name__      :       List[str]       						= os.path.join(
							    lowerCAmelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCAmelCase__			):
													copyfile(self.vocab_file      ,       lowerCAmelCase__			)

							return (out_vocab_file,)
