


"""simple docstring"""







import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowerCAmelCase__				    =       logging.getLogger(__name__)


@dataclass(frozen=lowercase    )
class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       a  :    str
       a  :    str
       a  :    Optional[str]       =None
       a  :    Optional[str]       =None
       a  :    Optional[str]       =None



@dataclass(frozen=lowercase    )
class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       a  :    List[int]
       a  :    Optional[List[int]]       =None
       a  :    Optional[List[int]]       =None
       a  :    Optional[Union[int, float]]       =None
       a  :    Optional[int]       =None


if is_torch_available():
     import torch
     from torch.utils.data import Dataset


     class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
            """simple docstring"""


            a  :    List[InputFeatures]

            def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__=False		,					snake_case__ = False		,					):






              """simple docstring"""





              lowerCAmelCase					:					str								=							hans_processors[task]()

              lowerCAmelCase					:					str								=							os.path.join(
                  snake_case__		,					"cached_{}_{}_{}_{}".format(
                      "dev" if evaluate else "train"		,					tokenizer.__class__.__name__		,					str(snake_case__						)		,					snake_case__		,					)		,					)
              lowerCAmelCase					:					Optional[Any]								=							processor.get_labels()
              if tokenizer.__class__ in (
                  RobertaTokenizer,
                  RobertaTokenizerFast,
                  XLMRobertaTokenizer,
                  BartTokenizer,
                  BartTokenizerFast,
              ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                lowerCAmelCase					:					Tuple								=							label_list[2], label_list[1]
              lowerCAmelCase					:					str								=							label_list

              # Make sure only the first process in distributed training processes the dataset,
              # and the others will use the cache.
              lowerCAmelCase					:					Dict								=							cached_features_file + ".lock"
              with FileLock(snake_case__						):
                if os.path.exists(snake_case__						) and not overwrite_cache:
                  logger.info(f"""Loading features from cached file {cached_features_file}"""						)
                  lowerCAmelCase					:					List[str]								=							torch.load(snake_case__						)
                else:
                  logger.info(f"""Creating features from dataset file at {data_dir}"""						)

                  lowerCAmelCase					:					Optional[Any]								=							(
                      processor.get_dev_examples(snake_case__						) if evaluate else processor.get_train_examples(snake_case__						)
                  )

                  logger.info("Training examples: %s"		,					len(snake_case__						)						)
                  lowerCAmelCase					:					List[str]								=							hans_convert_examples_to_features(snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						)
                  logger.info("Saving features into cached file %s"		,					snake_case__						)
                  torch.save(self.features		,					snake_case__						)

            def __len__(						self						):






              """simple docstring"""





              return len(self.features						)

            def __getitem__(						self		,					snake_case__						):






              """simple docstring"""





              return self.features[i]







            def 	lowercase__							(						self						):






              """simple docstring"""





              return self.label_list


if is_tf_available():
     import tensorflow as tf


     class 				SCREAMING_SNAKE_CASE__    :
            """simple docstring"""


            a  :    List[InputFeatures]

            def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = 128		,					snake_case__=False		,					snake_case__ = False		,					):






              """simple docstring"""





              lowerCAmelCase					:					Optional[int]								=							hans_processors[task]()
              lowerCAmelCase					:					Union[str, Any]								=							processor.get_labels()
              if tokenizer.__class__ in (
                  RobertaTokenizer,
                  RobertaTokenizerFast,
                  XLMRobertaTokenizer,
                  BartTokenizer,
                  BartTokenizerFast,
              ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                lowerCAmelCase					:					str								=							label_list[2], label_list[1]
              lowerCAmelCase					:					Union[str, Any]								=							label_list

              lowerCAmelCase					:					Tuple								=							processor.get_dev_examples(snake_case__						) if evaluate else processor.get_train_examples(snake_case__						)
              lowerCAmelCase					:					Optional[Any]								=							hans_convert_examples_to_features(snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						)

              def gen():
                for ex_index, ex in tqdm.tqdm(enumerate(self.features						)		,					desc="convert examples to features"						):
                  if ex_index % 10_000 == 0:
                    logger.info("Writing example %d of %d" % (ex_index, len(snake_case__						))						)

                  yield (
                      {
                          "example_id": 0,
                          "input_ids": ex.input_ids,
                          "attention_mask": ex.attention_mask,
                          "token_type_ids": ex.token_type_ids,
                      },
                      ex.label,
                  )

              lowerCAmelCase					:					int								=							tf.data.Dataset.from_generator(
                  snake_case__		,					(
                      {
                          "example_id": tf.intaa,
                          "input_ids": tf.intaa,
                          "attention_mask": tf.intaa,
                          "token_type_ids": tf.intaa,
                      },
                      tf.intaa,
                  )		,					(
                      {
                          "example_id": tf.TensorShape([]						),
                          "input_ids": tf.TensorShape([None, None]						),
                          "attention_mask": tf.TensorShape([None, None]						),
                          "token_type_ids": tf.TensorShape([None, None]						),
                      },
                      tf.TensorShape([]						),
                  )		,					)

            def 	lowercase__							(						self						):






              """simple docstring"""





              return self.dataset

            def __len__(						self						):






              """simple docstring"""





              return len(self.features						)

            def __getitem__(						self		,					snake_case__						):






              """simple docstring"""





              return self.features[i]







            def 	lowercase__							(						self						):






              """simple docstring"""





              return self.label_list


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         return self._create_examples(self._read_tsv(os.path.join(snake_case__		,					"heuristics_train_set.txt"						)						)		,					"train"						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         return self._create_examples(self._read_tsv(os.path.join(snake_case__		,					"heuristics_evaluation_set.txt"						)						)		,					"dev"						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         return ["contradiction", "entailment", "neutral"]







       def 	lowercase__							(						self		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[]
         for i, line in enumerate(snake_case__						):
           if i == 0:
             continue
           lowerCAmelCase					:					List[Any]								=							"%s-%s" % (set_type, line[0])
           lowerCAmelCase					:					Union[str, Any]								=							line[5]
           lowerCAmelCase					:					Dict								=							line[6]
           lowerCAmelCase					:					Tuple								=							line[7][2:] if line[7].startswith("ex"						) else line[7]
           lowerCAmelCase					:					List[str]								=							line[0]
           examples.append(InputExample(guid=snake_case__		,					text_a=snake_case__		,					text_b=snake_case__		,					label=snake_case__		,					pairID=snake_case__						)						)
         return examples



def  a__      (    SCREAMING_SNAKE_CASE   :   List[InputExample]    ,				SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   int    ,				SCREAMING_SNAKE_CASE   :   PreTrainedTokenizer    ,				):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							{label: i for i, label in enumerate(SCREAMING_SNAKE_CASE    )}

  lowerCAmelCase					:					List[str]								=							[]
  for ex_index, example in tqdm.tqdm(enumerate(SCREAMING_SNAKE_CASE    )    ,				desc="convert examples to features"    ):
    if ex_index % 1_0_0_0_0 == 0:
      logger.info("Writing example %d" % (ex_index)    )

    lowerCAmelCase					:					List[Any]								=							tokenizer(
        example.text_a    ,				example.text_b    ,				add_special_tokens=SCREAMING_SNAKE_CASE    ,				max_length=SCREAMING_SNAKE_CASE    ,				padding="max_length"    ,				truncation=SCREAMING_SNAKE_CASE    ,				return_overflowing_tokens=SCREAMING_SNAKE_CASE    ,				)

    lowerCAmelCase					:					Any								=							label_map[example.label] if example.label in label_map else 0

    lowerCAmelCase					:					Any								=							int(example.pairID    )

    features.append(InputFeatures(**SCREAMING_SNAKE_CASE    ,				label=SCREAMING_SNAKE_CASE    ,				pairID=SCREAMING_SNAKE_CASE    )    )

  for i, example in enumerate(examples[:5]    ):
    logger.info("*** Example ***"    )
    logger.info(f"""guid: {example}"""    )
    logger.info(f"""features: {features[i]}"""    )

  return features


lowerCAmelCase__				    =       {
    '''hans''': 3,
}





lowerCAmelCase__				    =       {
    '''hans''': HansProcessor,
}









"""simple docstring"""







import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible


class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             # Removed: 'unet/diffusion_pytorch_model.safetensors',
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             # Removed: 'text_encoder/model.safetensors',
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							[
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
         ]
         lowerCAmelCase					:					Dict								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
         ]
         lowerCAmelCase					:					int								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             # 'text_encoder/model.fp16.safetensors',
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					List[Any]								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)





