






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class       __SCREAMING_SNAKE_CASE   (				_UpperCAmelCase						,   unittest.TestCase):
      _SCREAMING_SNAKE_CASE		:    Optional[int]						=   RobertaTokenizer
      _SCREAMING_SNAKE_CASE		:    Union[str, Any]						=   RobertaTokenizerFast
      _SCREAMING_SNAKE_CASE		:    Any						=   True
      _SCREAMING_SNAKE_CASE		:    Any						=   {'''cls_token''': '''<s>'''}






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            lowerCAmelCase__        =  [
                """l""",
                """o""",
                """w""",
                """e""",
                """r""",
                """s""",
                """t""",
                """i""",
                """d""",
                """n""",
                """\u0120""",
                """\u0120l""",
                """\u0120n""",
                """\u0120lo""",
                """\u0120low""",
                """er""",
                """\u0120lowest""",
                """\u0120newer""",
                """\u0120wider""",
                """<unk>""",
            ]
            lowerCAmelCase__        =  dict(zip(lowercase_       ,  range(len(lowercase_				)				)				)				)
            lowerCAmelCase__        =  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
            lowerCAmelCase__        =  {"""unk_token""": """<unk>"""}

            lowerCAmelCase__        =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['vocab_file']				)
            lowerCAmelCase__        =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['merges_file']				)
            with open(self.vocab_file       ,  'w'       ,  encoding='utf-8'				) as fp:
                  fp.write(json.dumps(lowercase_				) + '\n'				)
            with open(self.merges_file       ,  'w'       ,  encoding='utf-8'				) as fp:
                  fp.write('\n'.join(lowercase_				)				)






      def  UpperCamelCase__ (		self       ,  **_UpperCamelCase				):






            """simple docstring"""
            kwargs.update(self.special_tokens_map				)
            return self.tokenizer_class.from_pretrained(self.tmpdirname       ,  **lowercase_				)






      def  UpperCamelCase__ (		self       ,  **_UpperCamelCase				):






            """simple docstring"""
            kwargs.update(self.special_tokens_map				)
            return RobertaTokenizerFast.from_pretrained(self.tmpdirname       ,  **lowercase_				)






      def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






            """simple docstring"""
            lowerCAmelCase__        =  """lower newer"""
            lowerCAmelCase__        =  """lower newer"""
            return input_text, output_text






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            lowerCAmelCase__        =  self.tokenizer_class(self.vocab_file       ,  self.merges_file       ,  **self.special_tokens_map				)
            lowerCAmelCase__        =  """lower newer"""
            lowerCAmelCase__        =  ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
            lowerCAmelCase__        =  tokenizer.tokenize(lowercase_				)  # , add_prefix_space=True)
            self.assertListEqual(lowercase_       ,  lowercase_				)

            lowerCAmelCase__        =  tokens + [tokenizer.unk_token]
            lowerCAmelCase__        =  [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_				)       ,  lowercase_				)






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            lowerCAmelCase__        =  self.get_tokenizer()

            self.assertListEqual(tokenizer.encode('Hello world!'       ,  add_special_tokens=lowercase_				)       ,  [0, 3_14_14, 2_32, 3_28, 2]				)
            self.assertListEqual(
                tokenizer.encode('Hello world! cécé herlolip 418'       ,  add_special_tokens=lowercase_				)       ,  [0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]       ,  )






      @slow
      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            lowerCAmelCase__        =  self.tokenizer_class.from_pretrained('roberta-base'				)

            lowerCAmelCase__        =  tokenizer.encode('sequence builders'       ,  add_special_tokens=lowercase_				)
            lowerCAmelCase__        =  tokenizer.encode('multi-sequence build'       ,  add_special_tokens=lowercase_				)

            lowerCAmelCase__        =  tokenizer.encode(
                'sequence builders'       ,  add_special_tokens=lowercase_       ,  add_prefix_space=lowercase_				)
            lowerCAmelCase__        =  tokenizer.encode(
                'sequence builders'       ,  'multi-sequence build'       ,  add_special_tokens=lowercase_       ,  add_prefix_space=lowercase_				)

            lowerCAmelCase__        =  tokenizer.build_inputs_with_special_tokens(lowercase_				)
            lowerCAmelCase__        =  tokenizer.build_inputs_with_special_tokens(lowercase_       ,  lowercase_				)

            assert encoded_sentence == encoded_text_from_decode
            assert encoded_pair == encoded_pair_from_decode






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            lowerCAmelCase__        =  self.get_tokenizer()

            lowerCAmelCase__        =  """Encode this sequence."""
            lowerCAmelCase__        =  tokenizer.byte_encoder[""" """.encode('utf-8'				)[0]]

            # Testing encoder arguments
            lowerCAmelCase__        =  tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_       ,  add_prefix_space=lowercase_				)
            lowerCAmelCase__        =  tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
            self.assertNotEqual(lowercase_       ,  lowercase_				)

            lowerCAmelCase__        =  tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_       ,  add_prefix_space=lowercase_				)
            lowerCAmelCase__        =  tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
            self.assertEqual(lowercase_       ,  lowercase_				)

            tokenizer.add_special_tokens({'bos_token': '<s>'}				)
            lowerCAmelCase__        =  tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_				)
            lowerCAmelCase__        =  tokenizer.convert_ids_to_tokens(encoded[1]				)[0]
            self.assertNotEqual(lowercase_       ,  lowercase_				)

            # Testing spaces after special tokens
            lowerCAmelCase__        =  """<mask>"""
            tokenizer.add_special_tokens(
                {'mask_token': AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_				)}				)  # mask token has a left space
            lowerCAmelCase__        =  tokenizer.convert_tokens_to_ids(lowercase_				)

            lowerCAmelCase__        =  """Encode <mask> sequence"""
            lowerCAmelCase__        =  """Encode <mask>sequence"""

            lowerCAmelCase__        =  tokenizer.encode(lowercase_				)
            lowerCAmelCase__        =  encoded.index(lowercase_				)
            lowerCAmelCase__        =  tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
            self.assertEqual(lowercase_       ,  lowercase_				)

            lowerCAmelCase__        =  tokenizer.encode(lowercase_				)
            lowerCAmelCase__        =  encoded.index(lowercase_				)
            lowerCAmelCase__        =  tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
            self.assertNotEqual(lowercase_       ,  lowercase_				)






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            pass






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(lowercase_       ,  **lowercase_				)
                        lowerCAmelCase__        =  self.tokenizer_class.from_pretrained(lowercase_       ,  **lowercase_				)
                        lowerCAmelCase__        =  """A, <mask> AllenNLP sentence."""
                        lowerCAmelCase__        =  tokenizer_r.encode_plus(lowercase_       ,  add_special_tokens=lowercase_       ,  return_token_type_ids=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_p.encode_plus(lowercase_       ,  add_special_tokens=lowercase_       ,  return_token_type_ids=lowercase_				)

                        # token_type_ids should put 0 everywhere
                        self.assertEqual(sum(tokens_r['token_type_ids']				)       ,  sum(tokens_p['token_type_ids']				)				)

                        # attention_mask should put 1 everywhere, so sum over length should be 1
                        self.assertEqual(
                            sum(tokens_r['attention_mask']				) / len(tokens_r['attention_mask']				)       ,  sum(tokens_p['attention_mask']				) / len(tokens_p['attention_mask']				)       ,  )

                        lowerCAmelCase__        =  tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']				)
                        lowerCAmelCase__        =  tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']				)

                        # Rust correctly handles the space before the mask while python doesnt
                        self.assertSequenceEqual(tokens_p['input_ids']       ,  [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)
                        self.assertSequenceEqual(tokens_r['input_ids']       ,  [0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)

                        self.assertSequenceEqual(
                            lowercase_       ,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
                        self.assertSequenceEqual(
                            lowercase_       ,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)






      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            for trim_offsets, add_prefix_space in itertools.product([True, False]       ,  repeat=2				):
                  lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                      self.tmpdirname       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)

                  lowerCAmelCase__        =  json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()				)
                  lowerCAmelCase__        =  json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()				)

                  self.assertEqual(pre_tokenizer_state['add_prefix_space']       ,  lowercase_				)

                  self.assertEqual(post_processor_state['add_prefix_space']       ,  lowercase_				)
                  self.assertEqual(post_processor_state['trim_offsets']       ,  lowercase_				)







      def  UpperCamelCase__ (		self				):






            """simple docstring"""
            # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
            # `trim_offsets`
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                        lowerCAmelCase__        =  """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                        lowerCAmelCase__        =  F"{text_of_1_token} {text_of_1_token}"

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (len(lowercase_				) + 1, len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (len(lowercase_				) + 1, len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (len(lowercase_				), len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (len(lowercase_				), len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  F" {text}"

                        # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                        #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                        # )
                        # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                        # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                        # self.assertEqual(
                        #     encoding.offset_mapping[1],
                        #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                        # )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (1, 1 + len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (1 + len(lowercase_				) + 1, 1 + len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, 1 + len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (1 + len(lowercase_				), 1 + len(lowercase_				) + 1 + len(lowercase_				))       ,  )

                        lowerCAmelCase__        =  self.rust_tokenizer_class.from_pretrained(
                            lowercase_       ,  use_fast=lowercase_       ,  add_prefix_space=lowercase_       ,  trim_offsets=lowercase_				)
                        lowerCAmelCase__        =  tokenizer_r(lowercase_       ,  return_offsets_mapping=lowercase_       ,  add_special_tokens=lowercase_				)
                        self.assertEqual(encoding.offset_mapping[0]       ,  (0, 1 + len(lowercase_				))				)
                        self.assertEqual(
                            encoding.offset_mapping[1]       ,  (1 + len(lowercase_				), 1 + len(lowercase_				) + 1 + len(lowercase_				))       ,  )













from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__snake_case      :      Union[str, Any]             =       {
    """configuration_convbert""": ["""CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ConvBertConfig""", """ConvBertOnnxConfig"""],
    """tokenization_convbert""": ["""ConvBertTokenizer"""],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case      :      List[str]             =       ["""ConvBertTokenizerFast"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case      :      Optional[Any]             =       [
					    """CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """ConvBertForMaskedLM""",
					    """ConvBertForMultipleChoice""",
					    """ConvBertForQuestionAnswering""",
					    """ConvBertForSequenceClassification""",
					    """ConvBertForTokenClassification""",
					    """ConvBertLayer""",
					    """ConvBertModel""",
					    """ConvBertPreTrainedModel""",
					    """load_tf_weights_in_convbert""",
					]


try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case      :      Union[str, Any]             =       [
					    """TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFConvBertForMaskedLM""",
					    """TFConvBertForMultipleChoice""",
					    """TFConvBertForQuestionAnswering""",
					    """TFConvBertForSequenceClassification""",
					    """TFConvBertForTokenClassification""",
					    """TFConvBertLayer""",
					    """TFConvBertModel""",
					    """TFConvBertPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
					from .tokenization_convbert import ConvBertTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_convbert_fast import ConvBertTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_convbert import (
										    CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    ConvBertForMaskedLM,
										    ConvBertForMultipleChoice,
										    ConvBertForQuestionAnswering,
										    ConvBertForSequenceClassification,
										    ConvBertForTokenClassification,
										    ConvBertLayer,
										    ConvBertModel,
										    ConvBertPreTrainedModel,
										    load_tf_weights_in_convbert,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_convbert import (
										    TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFConvBertForMaskedLM,
										    TFConvBertForMultipleChoice,
										    TFConvBertForQuestionAnswering,
										    TFConvBertForSequenceClassification,
										    TFConvBertForTokenClassification,
										    TFConvBertLayer,
										    TFConvBertModel,
										    TFConvBertPreTrainedModel,
										)


else:
					import sys

					__snake_case      :      Dict             =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





