




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 						_A (  _lowerCamelCase      ,       unittest.TestCase				):
  _UpperCamelCase       :      Any			=      RoCBertTokenizer
  _UpperCamelCase       :      Union[str, Any]			=      None
  _UpperCamelCase       :      Union[str, Any]			=      False
  _UpperCamelCase       :      List[Any]			=      True
  _UpperCamelCase       :      Union[str, Any]			=      filter_non_english






  def  __a      (  self		:     Tuple				)		->   List[Any]:






       """simple docstring"""







       super().setUp()

       lowercase :       Optional[Any]       =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
       lowercase :       Tuple       =			{}
       lowercase :       Dict       =			{}
       for i, value in enumerate(_A				):
            lowercase :       Any       =			i
            lowercase :       Dict       =			i
       lowercase :       List[Any]       =			os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''vocab_file''']				)
       lowercase :       Optional[int]       =			os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''word_shape_file''']				)
       lowercase :       Union[str, Any]       =			os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''word_pronunciation_file''']				)
       with open(self.vocab_file				,						'''w'''				,						encoding='''utf-8'''				) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)
       with open(self.word_shape_file				,						'''w'''				,						encoding='''utf-8'''				) as word_shape_writer:
            json.dump(_A				,						_A				,						ensure_ascii=_A				)
       with open(self.word_pronunciation_file				,						'''w'''				,						encoding='''utf-8'''				) as word_pronunciation_writer:
            json.dump(_A				,						_A				,						ensure_ascii=_A				)






  def  __a      (  self		:     List[Any]				)		->   List[str]:






       """simple docstring"""







       lowercase :       List[Any]       =			self.tokenizer_class(self.vocab_file				,						self.word_shape_file				,						self.word_pronunciation_file				)

       lowercase :       Tuple       =			tokenizer.tokenize('''你好[SEP]你是谁'''				)
       self.assertListEqual(_A				,						['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']				)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A				)				,						[5, 6, 2, 5, 7, 8]				)
       self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_A				)				,						[5, 6, 2, 5, 7, 8]				)
       self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_A				)				,						[5, 6, 2, 5, 7, 8]				)






  def  __a      (  self		:     Optional[Any]				)		->   str:






       """simple docstring"""







       lowercase :       str       =			RoCBertBasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''				)				,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']				)






  def  __a      (  self		:     List[Any]				)		->   Dict:






       """simple docstring"""







       lowercase :       List[str]       =			RoCBertBasicTokenizer(do_lower_case=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)				,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)				,						['''hello''']				)






  def  __a      (  self		:     Tuple				)		->   Dict:






       """simple docstring"""







       lowercase :       str       =			RoCBertBasicTokenizer(do_lower_case=_A				,						strip_accents=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)				,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)				,						['''h\u00E9llo''']				)






  def  __a      (  self		:     Optional[Any]				)		->   str:






       """simple docstring"""







       lowercase :       List[Any]       =			RoCBertBasicTokenizer(do_lower_case=_A				,						strip_accents=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)				,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)				,						['''hello''']				)






  def  __a      (  self		:     Tuple				)		->   Optional[int]:






       """simple docstring"""







       lowercase :       Optional[Any]       =			RoCBertBasicTokenizer(do_lower_case=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)				,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)				,						['''hello''']				)






  def  __a      (  self		:     Optional[int]				)		->   int:






       """simple docstring"""







       lowercase :       Optional[int]       =			RoCBertBasicTokenizer(do_lower_case=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)				,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)






  def  __a      (  self		:     Tuple				)		->   Tuple:






       """simple docstring"""







       lowercase :       Dict       =			RoCBertBasicTokenizer(do_lower_case=_A				,						strip_accents=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)				,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)






  def  __a      (  self		:     Optional[int]				)		->   Optional[Any]:






       """simple docstring"""







       lowercase :       int       =			RoCBertBasicTokenizer(do_lower_case=_A				,						strip_accents=_A				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)				,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)






  def  __a      (  self		:     str				)		->   int:






       """simple docstring"""







       lowercase :       Tuple       =			RoCBertBasicTokenizer(do_lower_case=_A				,						never_split=['''[UNK]''']				)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''				)				,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']				)






  def  __a      (  self		:     List[Any]				)		->   Dict:






       """simple docstring"""







       lowercase :       str       =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       lowercase :       Any       =			{}
       for i, token in enumerate(_A				):
            lowercase :       Optional[int]       =			i
       lowercase :       Dict       =			RoCBertWordpieceTokenizer(vocab=_A				,						unk_token='''[UNK]'''				)

       self.assertListEqual(tokenizer.tokenize(''''''				)				,						[]				)

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''				)				,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']				)

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''				)				,						['''[UNK]''', '''runn''', '''##ing''']				)






  def  __a      (  self		:     List[str]				)		->   Tuple:






       """simple docstring"""







       self.assertTrue(_is_whitespace(''' '''				)				)
       self.assertTrue(_is_whitespace('''\t'''				)				)
       self.assertTrue(_is_whitespace('''\r'''				)				)
       self.assertTrue(_is_whitespace('''\n'''				)				)
       self.assertTrue(_is_whitespace('''\u00A0'''				)				)

       self.assertFalse(_is_whitespace('''A'''				)				)
       self.assertFalse(_is_whitespace('''-'''				)				)






  def  __a      (  self		:     Optional[Any]				)		->   List[str]:






       """simple docstring"""







       self.assertTrue(_is_control('''\u0005'''				)				)

       self.assertFalse(_is_control('''A'''				)				)
       self.assertFalse(_is_control(''' '''				)				)
       self.assertFalse(_is_control('''\t'''				)				)
       self.assertFalse(_is_control('''\r'''				)				)






  def  __a      (  self		:     int				)		->   Tuple:






       """simple docstring"""







       self.assertTrue(_is_punctuation('''-'''				)				)
       self.assertTrue(_is_punctuation('''$'''				)				)
       self.assertTrue(_is_punctuation('''`'''				)				)
       self.assertTrue(_is_punctuation('''.'''				)				)

       self.assertFalse(_is_punctuation('''A'''				)				)
       self.assertFalse(_is_punctuation(''' '''				)				)






  def  __a      (  self		:     List[str]				)		->   List[str]:






       """simple docstring"""







       lowercase :       str       =			self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_A				) for t in ['''Test''', '''\xad''', '''test''']]				,						[['''[UNK]'''], [], ['''[UNK]''']]				)

       if self.test_rust_tokenizer:
            lowercase :       Dict       =			self.get_rust_tokenizer()
            self.assertListEqual(
                [rust_tokenizer.tokenize(_A				) for t in ['''Test''', '''\xad''', '''test''']]				,						[['''[UNK]'''], [], ['''[UNK]''']]				)






  def  __a      (  self		:     Union[str, Any]				)		->   Optional[int]:






       """simple docstring"""







       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                 lowercase :       str       =			self.rust_tokenizer_class.from_pretrained(_A				,						**_A				)

                 lowercase :       Tuple       =			f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                 lowercase :       Dict       =			tokenizer_r.encode_plus(
                     _A				,						return_attention_mask=_A				,						return_token_type_ids=_A				,						return_offsets_mapping=_A				,						add_special_tokens=_A				,						)

                 lowercase :       Any       =			tokenizer_r.do_lower_case if hasattr(_A				,						'''do_lower_case'''				) else False
                 lowercase :       Optional[int]       =			(
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''A'''),
                         ((1, 2), ''','''),
                         ((3, 5), '''na'''),
                         ((5, 6), '''##ï'''),
                         ((6, 8), '''##ve'''),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), '''Allen'''),
                         ((21, 23), '''##NL'''),
                         ((23, 24), '''##P'''),
                         ((25, 33), '''sentence'''),
                         ((33, 34), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''a'''),
                         ((1, 2), ''','''),
                         ((3, 8), '''naive'''),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), '''allen'''),
                         ((21, 23), '''##nl'''),
                         ((23, 24), '''##p'''),
                         ((25, 33), '''sentence'''),
                         ((33, 34), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]				,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']				)				)
                 self.assertEqual([e[0] for e in expected_results]				,						tokens['''offset_mapping''']				)






  def  __a      (  self		:     List[str]				)		->   Union[str, Any]:






       """simple docstring"""







       lowercase :       Dict       =			['''的''', '''人''', '''有''']
       lowercase :       Any       =			''''''.join(_A				)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                 lowercase :       Union[str, Any]       =			True
                 lowercase :       Dict       =			self.tokenizer_class.from_pretrained(_A				,						**_A				)
                 lowercase :       Tuple       =			self.rust_tokenizer_class.from_pretrained(_A				,						**_A				)

                 lowercase :       Optional[int]       =			tokenizer_p.encode(_A				,						add_special_tokens=_A				)
                 lowercase :       Optional[int]       =			tokenizer_r.encode(_A				,						add_special_tokens=_A				)

                 lowercase :       Any       =			tokenizer_r.convert_ids_to_tokens(_A				)
                 lowercase :       Union[str, Any]       =			tokenizer_p.convert_ids_to_tokens(_A				)

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(_A				,						_A				)
                 self.assertListEqual(_A				,						_A				)

                 lowercase :       Tuple       =			False
                 lowercase :       Optional[Any]       =			self.rust_tokenizer_class.from_pretrained(_A				,						**_A				)
                 lowercase :       Any       =			self.tokenizer_class.from_pretrained(_A				,						**_A				)

                 lowercase :       Union[str, Any]       =			tokenizer_r.encode(_A				,						add_special_tokens=_A				)
                 lowercase :       Dict       =			tokenizer_p.encode(_A				,						add_special_tokens=_A				)

                 lowercase :       Dict       =			tokenizer_r.convert_ids_to_tokens(_A				)
                 lowercase :       Union[str, Any]       =			tokenizer_p.convert_ids_to_tokens(_A				)

                 # it is expected that only the first Chinese character is not preceded by "##".
                 lowercase :       Any       =			[
                     f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A				)
                 ]
                 self.assertListEqual(_A				,						_A				)
                 self.assertListEqual(_A				,						_A				)






  @slow
  def  __a      (  self		:     Union[str, Any]				)		->   str:






       """simple docstring"""







       lowercase :       List[str]       =			self.tokenizer_class(self.vocab_file				,						self.word_shape_file				,						self.word_pronunciation_file				)

       lowercase :       Optional[int]       =			tokenizer.encode('''你好'''				,						add_special_tokens=_A				)
       lowercase :       Union[str, Any]       =			tokenizer.encode('''你是谁'''				,						add_special_tokens=_A				)

       lowercase :       List[Any]       =			tokenizer.build_inputs_with_special_tokens(_A				)
       lowercase :       Optional[Any]       =			tokenizer.build_inputs_with_special_tokens(_A				,						_A				)

       assert encoded_sentence == [1] + text + [2]
       assert encoded_pair == [1] + text + [2] + text_a + [2]






  def  __a      (  self		:     Dict				)		->   Optional[int]:






       """simple docstring"""







       lowercase :       Optional[Any]       =			self.get_tokenizers(do_lower_case=_A				)
       for tokenizer in tokenizers:
            with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
                 lowercase :       Union[str, Any]       =			'''你好，你是谁'''
                 lowercase :       List[Any]       =			tokenizer.tokenize(_A				)
                 lowercase :       Union[str, Any]       =			tokenizer.convert_tokens_to_ids(_A				)
                 lowercase :       int       =			tokenizer.convert_tokens_to_shape_ids(_A				)
                 lowercase :       Optional[int]       =			tokenizer.convert_tokens_to_pronunciation_ids(_A				)
                 lowercase :       Optional[Any]       =			tokenizer.prepare_for_model(
                     _A				,						_A				,						_A				,						add_special_tokens=_A				)

                 lowercase :       Optional[Any]       =			tokenizer.encode_plus(_A				,						add_special_tokens=_A				)

                 self.assertEqual(_A				,						_A				)





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_						=			logging.get_logger(__name__)

lowerCAmelCase_						=			{
    'facebook/xmod-base': 'https://huggingface.co/facebook/xmod-base/resolve/main/config.json',
    'facebook/xmod-large-prenorm': 'https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json',
    'facebook/xmod-base-13-125k': 'https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json',
    'facebook/xmod-base-30-125k': 'https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json',
    'facebook/xmod-base-30-195k': 'https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json',
    'facebook/xmod-base-60-125k': 'https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json',
    'facebook/xmod-base-60-265k': 'https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json',
    'facebook/xmod-base-75-125k': 'https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json',
    'facebook/xmod-base-75-269k': 'https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json',
}
class 						_A (  _lowerCamelCase				):
  _UpperCamelCase       :      Tuple			=      '''xmod'''






  def __init__(  self		:     Optional[Any]				,						_A		:     Union[str, Any]=30_522				,						_A		:     List[Any]=768				,						_A		:     Optional[Any]=12				,						_A		:     Any=12				,						_A		:     Tuple=3_072				,						_A		:     Optional[int]="gelu"				,						_A		:     List[Any]=0.1				,						_A		:     str=0.1				,						_A		:     List[Any]=512				,						_A		:     List[str]=2				,						_A		:     str=0.02				,						_A		:     Any=1E-12				,						_A		:     Union[str, Any]=1				,						_A		:     List[Any]=0				,						_A		:     Dict=2				,						_A		:     int="absolute"				,						_A		:     Dict=True				,						_A		:     int=None				,						_A		:     List[str]=False				,						_A		:     Dict=2				,						_A		:     int=False				,						_A		:     Optional[int]=True				,						_A		:     Any=True				,						_A		:     Optional[int]=("en_XX",)				,						_A		:     Any=None				,						**_A		:     Optional[int]				,						)		->   Union[str, Any]:






       """simple docstring"""







       super().__init__(pad_token_id=_A				,						bos_token_id=_A				,						eos_token_id=_A				,						**_A				)

       lowercase :       Optional[Any]       =			vocab_size
       lowercase :       Union[str, Any]       =			hidden_size
       lowercase :       Optional[Any]       =			num_hidden_layers
       lowercase :       Dict       =			num_attention_heads
       lowercase :       Union[str, Any]       =			hidden_act
       lowercase :       Tuple       =			intermediate_size
       lowercase :       List[str]       =			hidden_dropout_prob
       lowercase :       Union[str, Any]       =			attention_probs_dropout_prob
       lowercase :       Dict       =			max_position_embeddings
       lowercase :       Any       =			type_vocab_size
       lowercase :       Optional[Any]       =			initializer_range
       lowercase :       str       =			layer_norm_eps
       lowercase :       Tuple       =			position_embedding_type
       lowercase :       Optional[Any]       =			use_cache
       lowercase :       int       =			classifier_dropout
       lowercase :       Optional[int]       =			pre_norm
       lowercase :       Any       =			adapter_reduction_factor
       lowercase :       Union[str, Any]       =			adapter_layer_norm
       lowercase :       Optional[int]       =			adapter_reuse_layer_norm
       lowercase :       Optional[Any]       =			ln_before_adapter
       lowercase :       Union[str, Any]       =			list(_A				)
       lowercase :       List[Any]       =			default_language


class 						_A (  _lowerCamelCase				):






  @property
  def  __a      (  self		:     Union[str, Any]				)		->   Mapping[str, Mapping[int, str]]:






       """simple docstring"""







       if self.task == "multiple-choice":
            lowercase :       Any       =			{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
       else:
            lowercase :       Union[str, Any]       =			{0: '''batch''', 1: '''sequence'''}
       return OrderedDict(
           [
               ('''input_ids''', dynamic_axis),
               ('''attention_mask''', dynamic_axis),
           ]				)