





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

UpperCAmelCase     :     Tuple							=	logging.get_logger(__name__)


UpperCAmelCase     :     str							=	{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

UpperCAmelCase     :     Union[str, Any]							=	{
    """vocab_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"""},
    """merges_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"""},
    """tokenizer_config_file""": {
        """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"""
    },
}

UpperCAmelCase     :     Optional[int]							=	{"""facebook/blenderbot-3B""": 128}







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Dict            =      VOCAB_FILES_NAMES
      _lowercase    : str            =      PRETRAINED_VOCAB_FILES_MAP
      _lowercase    : str            =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase    : Dict            =      ["""input_ids""", """attention_mask"""]
      _lowercase    : Tuple            =      BlenderbotTokenizer







      def __init__(   self		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				lowerCAmelCase__="replace"		,				lowerCAmelCase__="<s>"		,				lowerCAmelCase__="</s>"		,				lowerCAmelCase__="</s>"		,				lowerCAmelCase__="<s>"		,				lowerCAmelCase__="<unk>"		,				lowerCAmelCase__="<pad>"		,				lowerCAmelCase__="<mask>"		,				lowerCAmelCase__=False		,				lowerCAmelCase__=True		,				**lowerCAmelCase__		,				)       ->						int:



        '''simple docstring'''




        super().__init__(
            lowerCAmelCase__		,				lowerCAmelCase__		,				tokenizer_file=lowerCAmelCase__		,				errors=lowerCAmelCase__		,				bos_token=lowerCAmelCase__		,				eos_token=lowerCAmelCase__		,				sep_token=lowerCAmelCase__		,				cls_token=lowerCAmelCase__		,				unk_token=lowerCAmelCase__		,				pad_token=lowerCAmelCase__		,				mask_token=lowerCAmelCase__		,				add_prefix_space=lowerCAmelCase__		,				trim_offsets=lowerCAmelCase__		,				**lowerCAmelCase__		,				)

        a__     : List[str]  =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
        if pre_tok_state.get("add_prefix_space"		,				lowerCAmelCase__							) != add_prefix_space:
          a__     : List[Any]  =getattr(lowerCAmelCase__		,				pre_tok_state.pop("type"							)							)
          a__     : Union[str, Any]  =add_prefix_space
          a__     : Dict  =pre_tok_class(**lowerCAmelCase__							)

        a__     : Dict  =add_prefix_space

        a__     : Optional[int]  ="post_processor"
        a__     : List[str]  =getattr(self.backend_tokenizer		,				lowerCAmelCase__		,				lowerCAmelCase__							)
        if tokenizer_component_instance:
          a__     : Optional[int]  =json.loads(tokenizer_component_instance.__getstate__()							)

          # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
          if "sep" in state:
            a__     : Dict  =tuple(state["sep"]							)
          if "cls" in state:
            a__     : List[Any]  =tuple(state["cls"]							)

          a__     : List[str]  =False

          if state.get("add_prefix_space"		,				lowerCAmelCase__							) != add_prefix_space:
            a__     : Union[str, Any]  =add_prefix_space
            a__     : Any  =True

          if state.get("trim_offsets"		,				lowerCAmelCase__							) != trim_offsets:
            a__     : int  =trim_offsets
            a__     : Optional[int]  =True

          if changes_to_apply:
            a__     : str  =getattr(lowerCAmelCase__		,				state.pop("type"							)							)
            a__     : Tuple  =component_class(**lowerCAmelCase__							)
            setattr(self.backend_tokenizer		,				lowerCAmelCase__		,				lowerCAmelCase__							)







      @property
      # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        if self._mask_token is None:
          if self.verbose:
            logger.error("Using mask_token, but it is not set yet."							)
          return None
        return str(self._mask_token							)







      @mask_token.setter
      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : int  =AddedToken(lowerCAmelCase__		,				lstrip=lowerCAmelCase__		,				rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__		,				lowerCAmelCase__							) else value
        a__     : Union[str, Any]  =value







      def        _lowercase  (   self		,				*lowerCAmelCase__		,				**lowerCAmelCase__							)       ->						BatchEncoding:



        '''simple docstring'''




        a__     : int  =kwargs.get("is_split_into_words"		,				lowerCAmelCase__							)
        assert self.add_prefix_space or not is_split_into_words, (
            F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*lowerCAmelCase__		,				**lowerCAmelCase__							)







      def        _lowercase  (   self		,				*lowerCAmelCase__		,				**lowerCAmelCase__							)       ->						BatchEncoding:



        '''simple docstring'''




        a__     : Dict  =kwargs.get("is_split_into_words"		,				lowerCAmelCase__							)

        assert self.add_prefix_space or not is_split_into_words, (
            F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*lowerCAmelCase__		,				**lowerCAmelCase__							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__ = None							)       ->						Tuple[str]:



        '''simple docstring'''




        a__     : Union[str, Any]  =self._tokenizer.model.save(lowerCAmelCase__		,				name=lowerCAmelCase__							)
        return tuple(lowerCAmelCase__							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__ = None							)       ->						List[int]:



        '''simple docstring'''




        a__     : Any  =[self.sep_token_id]
        a__     : Optional[Any]  =[self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep							) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__ = None							)       ->						Optional[Any]:



        '''simple docstring'''




        return token_ids_a + [self.eos_token_id]







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						List[int]:



        '''simple docstring'''




        a__     : int  =[]
        for is_user, text in conversation.iter_texts():
          if is_user:
            # We need to space prefix as it's being done within blenderbot
            inputs.append(" " + text							)
          else:
            # Generated responses should contain them already.
            inputs.append(lowerCAmelCase__							)

        a__     : List[str]  ="  ".join(lowerCAmelCase__							)
        a__     : Union[str, Any]  =self.encode(lowerCAmelCase__							)
        if len(lowerCAmelCase__							) > self.model_max_length:
          a__     : int  =input_ids[-self.model_max_length :]
          logger.warning(F'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''							)
        return input_ids













import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
UpperCAmelCase     :     List[Any]							=	{"""LayoutLMv2Config""", """LayoutLMv3Config"""}







@is_pipeline_test
class     __lowerCAmelCase	(   unittest.TestCase):
      _lowercase    : str            =      MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
      _lowercase    : Tuple            =      TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

      if model_mapping is not None:
            _lowercase    : Dict            =      {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
      if tf_model_mapping is not None:
            _lowercase    : int            =      {
                config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
            }







      @require_torch
      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        a__     : str  =pipeline(
            task="text-classification"		,				model="hf-internal-testing/tiny-random-distilbert"		,				framework="pt"							)

        a__     : Any  =text_classifier("This is great !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}]							)

        a__     : Any  =text_classifier("This is great !"		,				top_k=2							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}]							)

        a__     : Tuple  =text_classifier(["This is great !", "This is bad"]		,				top_k=2							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[
                [{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}],
                [{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}],
            ]		,				)

        a__     : List[Any]  =text_classifier("This is great !"		,				top_k=1							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}]							)

        # Legacy behavior
        a__     : Any  =text_classifier("This is great !"		,				return_all_scores=lowerCAmelCase__							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}]							)

        a__     : List[str]  =text_classifier("This is great !"		,				return_all_scores=lowerCAmelCase__							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[[{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}]]							)

        a__     : Optional[int]  =text_classifier(["This is great !", "Something else"]		,				return_all_scores=lowerCAmelCase__							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[
                [{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}],
                [{"label": "LABEL_0", "score": 0.5_04}, {"label": "LABEL_1", "score": 0.4_96}],
            ]		,				)

        a__     : int  =text_classifier(["This is great !", "Something else"]		,				return_all_scores=lowerCAmelCase__							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[
                {"label": "LABEL_0", "score": 0.5_04},
                {"label": "LABEL_0", "score": 0.5_04},
            ]		,				)







      @require_torch
      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        import torch

        a__     : Dict  =pipeline(
            task="text-classification"		,				model="hf-internal-testing/tiny-random-distilbert"		,				framework="pt"		,				device=torch.device("cpu"							)		,				)

        a__     : Optional[Any]  =text_classifier("This is great !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}]							)







      @require_tf
      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : Optional[int]  =pipeline(
            task="text-classification"		,				model="hf-internal-testing/tiny-random-distilbert"		,				framework="tf"							)

        a__     : Optional[Any]  =text_classifier("This is great !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "LABEL_0", "score": 0.5_04}]							)







      @slow
      @require_torch
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : Union[str, Any]  =pipeline("text-classification"							)

        a__     : Union[str, Any]  =text_classifier("This is great !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "POSITIVE", "score": 1.0}]							)
        a__     : Optional[Any]  =text_classifier("This is bad !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "NEGATIVE", "score": 1.0}]							)
        a__     : Dict  =text_classifier("Birds are a type of animal"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "POSITIVE", "score": 0.9_88}]							)







      @slow
      @require_tf
      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Tuple  =pipeline("text-classification"		,				framework="tf"							)

        a__     : str  =text_classifier("This is great !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "POSITIVE", "score": 1.0}]							)
        a__     : str  =text_classifier("This is bad !"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "NEGATIVE", "score": 1.0}]							)
        a__     : Dict  =text_classifier("Birds are a type of animal"							)
        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": "POSITIVE", "score": 0.9_88}]							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : List[Any]  =TextClassificationPipeline(model=lowerCAmelCase__		,				tokenizer=lowerCAmelCase__							)
        return text_classifier, ["HuggingFace is in", "This is another test"]







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						List[str]:



        '''simple docstring'''




        a__     : Tuple  =text_classifier.model
        # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
        a__     : List[Any]  ="HuggingFace is in"
        a__     : int  =text_classifier(lowerCAmelCase__							)

        self.assertEqual(nested_simplify(lowerCAmelCase__							)		,				[{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}]							)
        self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()							)

        a__     : Optional[int]  =["HuggingFace is in ", "Paris is in France"]
        a__     : Optional[Any]  =text_classifier(lowerCAmelCase__							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}, {"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}]		,				)
        self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()							)
        self.assertTrue(outputs[1]["label"] in model.config.idalabel.values()							)

        # Forcing to get all results with `top_k=None`
        # This is NOT the legacy format
        a__     : Union[str, Any]  =text_classifier(lowerCAmelCase__		,				top_k=lowerCAmelCase__							)
        a__     : Optional[Any]  =len(model.config.idalabel.values()							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[[{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}] * N, [{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}] * N]		,				)

        a__     : List[str]  ={"text": "HuggingFace is in ", "text_pair": "Paris is in France"}
        a__     : Optional[Any]  =text_classifier(lowerCAmelCase__							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}		,				)
        self.assertTrue(outputs["label"] in model.config.idalabel.values()							)

        # This might be used a text pair, but tokenizer + pipe interaction
        # makes it hard to understand that it's not using the pair properly
        # https://github.com/huggingface/transformers/issues/17305
        # We disabled this usage instead as it was outputting wrong outputs.
        a__     : Any  =[["HuggingFace is in ", "Paris is in France"]]
        with self.assertRaises(lowerCAmelCase__							):
          text_classifier(lowerCAmelCase__							)

        # This used to be valid for doing text pairs
        # We're keeping it working because of backward compatibility
        a__     : Optional[int]  =text_classifier([[["HuggingFace is in ", "Paris is in France"]]]							)
        self.assertEqual(
            nested_simplify(lowerCAmelCase__							)		,				[{"label": ANY(lowerCAmelCase__							), "score": ANY(lowerCAmelCase__							)}]		,				)
        self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()							)






