



'''simple docstring'''






import argparse
import os
import re

import packaging.version


lowerCAmelCase: List[str]							=						'examples/'
lowerCAmelCase: List[Any]							=						{
    'examples': (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'),
    'doc': (re.compile(r'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
lowerCAmelCase: str							=						{
    'init': 'src/transformers/__init__.py',
    'setup': 'setup.py',
}
lowerCAmelCase: str							=						'README.md'
def 				lowerCamelCase__	(						_A			,      _A			,      _A    ):
  with open(_A			,      'r'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    a						: Tuple					       =					f.read()
  a       ,     a						: Tuple					       =					REPLACE_PATTERNS[pattern]
  a						: Dict					       =					replace.replace('VERSION'			,      _A    )
  a						: Dict					       =					re_pattern.sub(_A			,      _A    )
  with open(_A			,      'w'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    f.write(_A    )
def 				lowerCamelCase__	(						_A    ):
  for folder, directories, fnames in os.walk(_A    ):
    # Removing some of the folders with non-actively maintained examples from the walk
    if "research_projects" in directories:
      directories.remove('research_projects'    )
    if "legacy" in directories:
      directories.remove('legacy'    )
    for fname in fnames:
      if fname.endswith('.py'    ):
        update_version_in_file(os.path.join(_A			,      _A    )			,      _A			,      pattern='examples'    )
def 				lowerCamelCase__	(						_A			,      _A=False    ):
  for pattern, fname in REPLACE_FILES.items():
    update_version_in_file(_A			,      _A			,      _A    )
  if not patch:
    update_version_in_examples(_A    )
def 				lowerCamelCase__	(						):
  a						: Tuple					       =					'🤗 Transformers currently provides the following architectures'
  a						: Any					       =					'1. Want to contribute a new model?'
  with open(_A			,      'r'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    a						: Tuple					       =					f.readlines()

  # Find the start of the list.
  a						: Optional[int]					       =					0
  while not lines[start_index].startswith(_start_prompt    ):
    start_index += 1
  start_index += 1

  a						: Optional[int]					       =					start_index
  # Update the lines in the model list.
  while not lines[index].startswith(_end_prompt    ):
    if lines[index].startswith('1.'    ):
      a						: List[Any]					       =					lines[index].replace(
          'https://huggingface.co/docs/transformers/main/model_doc'			,      'https://huggingface.co/docs/transformers/model_doc'			,      )
    index += 1

  with open(_A			,      'w'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    f.writelines(_A    )
def 				lowerCamelCase__	(						):
  with open(REPLACE_FILES['init']			,      'r'    ) as f:
    a						: Union[str, Any]					       =					f.read()
  a						: Tuple					       =					REPLACE_PATTERNS['init'][0].search(_A    ).groups()[0]
  return packaging.version.parse(_A    )
def 				lowerCamelCase__	(						_A=False    ):
  a						: int					       =					get_version()
  if patch and default_version.is_devrelease:
    raise ValueError('Can\'t create a patch version from the dev branch, checkout a released version!'    )
  if default_version.is_devrelease:
    a						: Any					       =					default_version.base_version
  elif patch:
    a						: Dict					       =					f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
  else:
    a						: Union[str, Any]					       =					f"""{default_version.major}.{default_version.minor + 1}.0"""

  # Now let's ask nicely if that's the right one.
  a						: List[Any]					       =					input(f"""Which version are you releasing? [{default_version}]"""    )
  if len(_A    ) == 0:
    a						: Union[str, Any]					       =					default_version

  print(f"""Updating version to {version}."""    )
  global_version_update(_A			,      patch=_A    )
  if not patch:
    print('Cleaning main README, don\'t forget to run `make fix-copies`.'    )
    clean_main_ref_in_model_list()
def 				lowerCamelCase__	(						):
  a						: int					       =					get_version()
  a						: Any					       =					f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
  a						: int					       =					current_version.base_version

  # Check with the user we got that right.
  a						: Tuple					       =					input(f"""Which version are we developing now? [{dev_version}]"""    )
  if len(_A    ) == 0:
    a						: Optional[int]					       =					dev_version

  print(f"""Updating version to {version}."""    )
  global_version_update(_A    )
  print('Cleaning main README, don\'t forget to run `make fix-copies`.'    )
  clean_main_ref_in_model_list()


if __name__ == "__main__":
     lowerCAmelCase: Tuple							=						argparse.ArgumentParser()
     parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
     parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
     lowerCAmelCase: Optional[Any]							=						parser.parse_args()
     if not args.post_release:
          pre_release_work(patch=args.patch)
     elif args.patch:
          print('Nothing to do after a patch :-)')
     else:
          post_release_work()




'''simple docstring'''






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 							a__( lowerCamelCase__		,			unittest.TestCase     ):
    lowercase__			=   BertJapaneseTokenizer
    lowercase__			=   False
    lowercase__			=   True







    def       lowercase_	(				self			:  int     ):
      super().setUp()

      a						: List[Any]					       =					[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          'こんにちは',
          'こん',
          'にちは',
          'ばんは',
          '##こん',
          '##にちは',
          '##ばんは',
          '世界',
          '##世界',
          '、',
          '##、',
          '。',
          '##。',
      ]

      a						: Dict					       =					os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['vocab_file']     )
      with open(self.vocab_file						,      'w'						,      encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )







    def       lowercase_	(				self			:  Any						,      __snake_case			:  str     ):
      a						: Union[str, Any]					       =					'こんにちは、世界。 \nこんばんは、世界。'
      a						: List[Any]					       =					'こんにちは 、 世界 。 こんばんは 、 世界 。'
      return input_text, output_text







    def       lowercase_	(				self			:  Optional[Any]						,      __snake_case			:  Optional[Any]     ):
      a       ,     a						: List[str]					       =					self.get_input_output_texts(__snake_case     )
      a						: Optional[int]					       =					tokenizer.encode(__snake_case						,      add_special_tokens=__snake_case     )
      a						: str					       =					tokenizer.decode(__snake_case						,      clean_up_tokenization_spaces=__snake_case     )
      return text, ids







    def       lowercase_	(				self			:  Optional[Any]     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  List[Any]     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  Dict     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  List[Any]     ):
      a						: Optional[int]					       =					self.tokenizer_class(self.vocab_file     )

      a						: Optional[int]					       =					tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'     )
      self.assertListEqual(__snake_case						,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case     )						,      [3, 12, 10, 14, 4, 9, 12, 10, 14]     )







    def       lowercase_	(				self			:  Union[str, Any]     ):
      a						: Tuple					       =					self.tokenizer_class(self.vocab_file						,      word_tokenizer_type='mecab'     )
      self.assertIsNotNone(__snake_case     )

      a						: List[str]					       =					'こんにちは、世界。\nこんばんは、世界。'
      a						: Tuple					       =					tokenizer.tokenize(__snake_case     )
      self.assertListEqual(__snake_case						,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case     )						,      [3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      a						: Optional[int]					       =					os.path.join(self.tmpdirname						,      'tokenizer.bin'     )
      with open(__snake_case						,      'wb'     ) as handle:
        pickle.dump(__snake_case						,      __snake_case     )

      with open(__snake_case						,      'rb'     ) as handle:
        a						: Optional[Any]					       =					pickle.load(__snake_case     )

      a						: Tuple					       =					tokenizer_new.tokenize(__snake_case     )

      self.assertListEqual(__snake_case						,      __snake_case     )







    def       lowercase_	(				self			:  Dict     ):
      a						: List[str]					       =					MecabTokenizer(mecab_dic='ipadic'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,      )







    def       lowercase_	(				self			:  List[Any]     ):
      try:
        a						: int					       =					MecabTokenizer(mecab_dic='unidic_lite'     )
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,      )







    def       lowercase_	(				self			:  Any     ):
      try:
        a						: Union[str, Any]					       =					MecabTokenizer(mecab_dic='unidic'     )
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,      )







    def       lowercase_	(				self			:  str     ):
      a						: Tuple					       =					MecabTokenizer(do_lower_case=__snake_case						,      mecab_dic='ipadic'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,      )







    def       lowercase_	(				self			:  Union[str, Any]     ):
      try:
        a						: Any					       =					MecabTokenizer(
            do_lower_case=__snake_case						,      normalize_text=__snake_case						,      mecab_option='-d /usr/local/lib/mecab/dic/jumandic'     )
      except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']						,      )







    def       lowercase_	(				self			:  List[Any]     ):
      a						: Dict					       =					MecabTokenizer(normalize_text=__snake_case						,      mecab_dic='ipadic'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']						,      )







    @require_sudachi
    def       lowercase_	(				self			:  str     ):
      a						: Optional[int]					       =					self.tokenizer_class(self.vocab_file						,      word_tokenizer_type='sudachi'     )
      self.assertIsNotNone(__snake_case     )

      a						: List[Any]					       =					'こんにちは、世界。\nこんばんは、世界。'
      a						: int					       =					tokenizer.tokenize(__snake_case     )
      self.assertListEqual(__snake_case						,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case     )						,      [3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      a						: Tuple					       =					os.path.join(self.tmpdirname						,      'tokenizer.bin'     )
      with open(__snake_case						,      'wb'     ) as handle:
        pickle.dump(__snake_case						,      __snake_case     )

      with open(__snake_case						,      'rb'     ) as handle:
        a						: Optional[int]					       =					pickle.load(__snake_case     )

      a						: List[Any]					       =					tokenizer_new.tokenize(__snake_case     )

      self.assertListEqual(__snake_case						,      __snake_case     )







    @require_sudachi
    def       lowercase_	(				self			:  List[Any]     ):
      a						: Optional[Any]					       =					SudachiTokenizer(sudachi_dict_type='core'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']						,      )







    @require_sudachi
    def       lowercase_	(				self			:  Any     ):
      a						: str					       =					SudachiTokenizer(sudachi_dict_type='core'						,      sudachi_split_mode='A'     )

      self.assertListEqual(tokenizer.tokenize('外国人参政権'     )						,      ['外国', '人', '参政', '権']     )







    @require_sudachi
    def       lowercase_	(				self			:  Optional[Any]     ):
      a						: Optional[int]					       =					SudachiTokenizer(sudachi_dict_type='core'						,      sudachi_split_mode='B'     )

      self.assertListEqual(tokenizer.tokenize('外国人参政権'     )						,      ['外国人', '参政権']     )







    @require_sudachi
    def       lowercase_	(				self			:  Optional[Any]     ):
      a						: Dict					       =					SudachiTokenizer(sudachi_dict_type='core'						,      sudachi_split_mode='C'     )

      self.assertListEqual(tokenizer.tokenize('外国人参政権'     )						,      ['外国人参政権']     )







    @require_sudachi
    def       lowercase_	(				self			:  Dict     ):
      a						: Optional[int]					       =					SudachiTokenizer(do_lower_case=__snake_case						,      sudachi_dict_type='core'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']						,      )







    @require_sudachi
    def       lowercase_	(				self			:  Tuple     ):
      a						: int					       =					SudachiTokenizer(normalize_text=__snake_case						,      sudachi_dict_type='core'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']						,      )







    @require_sudachi
    def       lowercase_	(				self			:  Union[str, Any]     ):
      a						: List[str]					       =					SudachiTokenizer(trim_whitespace=__snake_case						,      sudachi_dict_type='core'     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']						,      )







    @require_jumanpp
    def       lowercase_	(				self			:  List[Any]     ):
      a						: Optional[int]					       =					self.tokenizer_class(self.vocab_file						,      word_tokenizer_type='jumanpp'     )
      self.assertIsNotNone(__snake_case     )

      a						: str					       =					'こんにちは、世界。\nこんばんは、世界。'
      a						: Tuple					       =					tokenizer.tokenize(__snake_case     )
      self.assertListEqual(__snake_case						,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case     )						,      [3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      a						: Optional[Any]					       =					os.path.join(self.tmpdirname						,      'tokenizer.bin'     )
      with open(__snake_case						,      'wb'     ) as handle:
        pickle.dump(__snake_case						,      __snake_case     )

      with open(__snake_case						,      'rb'     ) as handle:
        a						: List[str]					       =					pickle.load(__snake_case     )

      a						: Any					       =					tokenizer_new.tokenize(__snake_case     )

      self.assertListEqual(__snake_case						,      __snake_case     )







    @require_jumanpp
    def       lowercase_	(				self			:  List[str]     ):
      a						: Any					       =					JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,      )







    @require_jumanpp
    def       lowercase_	(				self			:  List[str]     ):
      a						: List[Any]					       =					JumanppTokenizer(do_lower_case=__snake_case     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,      )







    @require_jumanpp
    def       lowercase_	(				self			:  Any     ):
      a						: List[Any]					       =					JumanppTokenizer(normalize_text=__snake_case     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']						,      )







    @require_jumanpp
    def       lowercase_	(				self			:  Any     ):
      a						: str					       =					JumanppTokenizer(trim_whitespace=__snake_case     )

      self.assertListEqual(
          tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )						,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']						,      )







    @require_jumanpp
    def       lowercase_	(				self			:  Tuple     ):
      a						: int					       =					JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'     )						,      ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']						,      )







    def       lowercase_	(				self			:  Any     ):
      a						: int					       =					['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

      a						: Optional[int]					       =					{}
      for i, token in enumerate(__snake_case     ):
        a						: Dict					       =					i
      a						: Optional[Any]					       =					WordpieceTokenizer(vocab=__snake_case						,      unk_token='[UNK]'     )

      self.assertListEqual(tokenizer.tokenize(''     )						,      []     )

      self.assertListEqual(tokenizer.tokenize('こんにちは'     )						,      ['こんにちは']     )

      self.assertListEqual(tokenizer.tokenize('こんばんは'     )						,      ['こん', '##ばんは']     )

      self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'     )						,      ['こん', '##ばんは', '[UNK]', 'こんにちは']     )







    def       lowercase_	(				self			:  Tuple     ):
      a						: List[Any]					       =					BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'     )
      a						: List[Any]					       =					tokenizer.subword_tokenizer

      a						: List[str]					       =					subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'     )
      self.assertListEqual(__snake_case						,      ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']     )

      a						: Union[str, Any]					       =					subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'     )
      self.assertListEqual(__snake_case						,      ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']     )







    def       lowercase_	(				self			:  Union[str, Any]     ):
      a						: Optional[Any]					       =					self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'     )

      a						: Dict					       =					tokenizer.encode('ありがとう。'						,      add_special_tokens=__snake_case     )
      a						: str					       =					tokenizer.encode('どういたしまして。'						,      add_special_tokens=__snake_case     )

      a						: Optional[int]					       =					tokenizer.build_inputs_with_special_tokens(__snake_case     )
      a						: Union[str, Any]					       =					tokenizer.build_inputs_with_special_tokens(__snake_case						,      __snake_case     )

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]

@custom_tokenizers
class 							a__( lowerCamelCase__		,			unittest.TestCase     ):
    lowercase__			=   BertJapaneseTokenizer
    lowercase__			=   False







    def       lowercase_	(				self			:  List[Any]     ):
      super().setUp()

      a						: List[Any]					       =					['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

      a						: List[Any]					       =					os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['vocab_file']     )
      with open(self.vocab_file						,      'w'						,      encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )







    def       lowercase_	(				self			:  Optional[Any]						,      **__snake_case			:  List[Any]     ):
      return BertJapaneseTokenizer.from_pretrained(self.tmpdirname						,      subword_tokenizer_type='character'						,      **__snake_case     )







    def       lowercase_	(				self			:  Tuple						,      __snake_case			:  List[str]     ):
      a						: int					       =					'こんにちは、世界。 \nこんばんは、世界。'
      a						: Optional[Any]					       =					'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
      return input_text, output_text







    def       lowercase_	(				self			:  str     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  List[str]     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  Any     ):
      pass  # TODO add if relevant







    def       lowercase_	(				self			:  Any     ):
      a						: Optional[int]					       =					self.tokenizer_class(self.vocab_file						,      subword_tokenizer_type='character'     )

      a						: Tuple					       =					tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'     )
      self.assertListEqual(
          __snake_case						,      ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']     )
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__snake_case     )						,      [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]     )







    def       lowercase_	(				self			:  Any     ):
      a						: Union[str, Any]					       =					['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

      a						: Optional[Any]					       =					{}
      for i, token in enumerate(__snake_case     ):
        a						: Tuple					       =					i
      a						: Optional[int]					       =					CharacterTokenizer(vocab=__snake_case						,      unk_token='[UNK]'     )

      self.assertListEqual(tokenizer.tokenize(''     )						,      []     )

      self.assertListEqual(tokenizer.tokenize('こんにちは'     )						,      ['こ', 'ん', 'に', 'ち', 'は']     )

      self.assertListEqual(tokenizer.tokenize('こんにちほ'     )						,      ['こ', 'ん', 'に', 'ち', '[UNK]']     )







    def       lowercase_	(				self			:  Tuple     ):
      a						: List[Any]					       =					self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'     )

      a						: Optional[int]					       =					tokenizer.encode('ありがとう。'						,      add_special_tokens=__snake_case     )
      a						: List[str]					       =					tokenizer.encode('どういたしまして。'						,      add_special_tokens=__snake_case     )

      a						: Optional[int]					       =					tokenizer.build_inputs_with_special_tokens(__snake_case     )
      a						: Dict					       =					tokenizer.build_inputs_with_special_tokens(__snake_case						,      __snake_case     )

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]

@custom_tokenizers
class 							a__( unittest.TestCase     ):







    def       lowercase_	(				self			:  List[str]     ):
      a						: List[Any]					       =					'cl-tohoku/bert-base-japanese'
      a						: Dict					       =					AutoTokenizer.from_pretrained(__snake_case     )
      self.assertIsInstance(__snake_case						,      __snake_case     )

class 							a__( unittest.TestCase     ):







    def       lowercase_	(				self			:  Union[str, Any]     ):
      a						: List[str]					       =					'cl-tohoku/bert-base-japanese'
      with self.assertLogs('transformers'						,      level='WARNING'     ) as cm:
        BertTokenizer.from_pretrained(__snake_case     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )
      a						: Dict					       =					'bert-base-cased'
      with self.assertLogs('transformers'						,      level='WARNING'     ) as cm:
        BertJapaneseTokenizer.from_pretrained(__snake_case     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )