


"""simple docstring"""



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class _lowerCAmelCase    (				__lowercase   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	List[Any]							  =  BertJapaneseTokenizer
   __UpperCAmelCase			:	Any							  =  False
   __UpperCAmelCase			:	List[Any]							  =  True







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       super().setUp()

       snake_case   :				int				 =   [
           """[UNK]""",
           """[CLS]""",
           """[SEP]""",
           """こんにちは""",
           """こん""",
           """にちは""",
           """ばんは""",
           """##こん""",
           """##にちは""",
           """##ばんは""",
           """世界""",
           """##世界""",
           """、""",
           """##、""",
           """。""",
           """##。""",
       ]

       snake_case   :				Optional[Any]				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["vocab_file"]						)
       with open(self.vocab_file						,     "w"						,     encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   """こんにちは、世界。 \nこんばんは、世界。"""
       snake_case   :				str				 =   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
       return input_text, output_text







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   self.get_input_output_texts(UpperCamelCase__						)
       snake_case   :				List[str]				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
       snake_case   :				Optional[int]				 =   tokenizer.decode(UpperCamelCase__						,     clean_up_tokenization_spaces=UpperCamelCase__						)
       return text, ids







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   self.tokenizer_class(self.vocab_file						)

       snake_case   :				str				 =   tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"						)
       self.assertListEqual(UpperCamelCase__						,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   self.tokenizer_class(self.vocab_file						,     word_tokenizer_type="mecab"						)
       self.assertIsNotNone(UpperCamelCase__						)

       snake_case   :				int				 =   """こんにちは、世界。\nこんばんは、世界。"""
       snake_case   :				List[str]				 =   tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       snake_case   :				Optional[int]				 =   os.path.join(self.tmpdirname						,     "tokenizer.bin"						)
       with open(UpperCamelCase__						,     "wb"						) as handle:
           pickle.dump(UpperCamelCase__						,     UpperCamelCase__						)

       with open(UpperCamelCase__						,     "rb"						) as handle:
           snake_case   :				Tuple				 =   pickle.load(UpperCamelCase__						)

       snake_case   :				Dict				 =   tokenizer_new.tokenize(UpperCamelCase__						)

       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Tuple				 =   MecabTokenizer(mecab_dic="ipadic"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       try:
           snake_case   :				List[str]				 =   MecabTokenizer(mecab_dic="unidic_lite"						)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       try:
           snake_case   :				Optional[Any]				 =   MecabTokenizer(mecab_dic="unidic"						)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				str				 =   MecabTokenizer(do_lower_case=UpperCamelCase__						,     mecab_dic="ipadic"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       try:
           snake_case   :				Optional[Any]				 =   MecabTokenizer(
               do_lower_case=UpperCamelCase__						,     normalize_text=UpperCamelCase__						,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"						)
       except RuntimeError:
           # if dict doesn't exist in the system, previous code raises this error.
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   MecabTokenizer(normalize_text=UpperCamelCase__						,     mecab_dic="ipadic"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]						,     )







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				str				 =   self.tokenizer_class(self.vocab_file						,     word_tokenizer_type="sudachi"						)
       self.assertIsNotNone(UpperCamelCase__						)

       snake_case   :				Any				 =   """こんにちは、世界。\nこんばんは、世界。"""
       snake_case   :				List[Any]				 =   tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       snake_case   :				Tuple				 =   os.path.join(self.tmpdirname						,     "tokenizer.bin"						)
       with open(UpperCamelCase__						,     "wb"						) as handle:
           pickle.dump(UpperCamelCase__						,     UpperCamelCase__						)

       with open(UpperCamelCase__						,     "rb"						) as handle:
           snake_case   :				int				 =   pickle.load(UpperCamelCase__						)

       snake_case   :				str				 =   tokenizer_new.tokenize(UpperCamelCase__						)

       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   SudachiTokenizer(sudachi_dict_type="core"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]						,     )







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				str				 =   SudachiTokenizer(sudachi_dict_type="core"						,     sudachi_split_mode="A"						)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"						)						,     ["外国", "人", "参政", "権"]						)







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				Dict				 =   SudachiTokenizer(sudachi_dict_type="core"						,     sudachi_split_mode="B"						)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"						)						,     ["外国人", "参政権"]						)







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   SudachiTokenizer(sudachi_dict_type="core"						,     sudachi_split_mode="C"						)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"						)						,     ["外国人参政権"]						)







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				int				 =   SudachiTokenizer(do_lower_case=UpperCamelCase__						,     sudachi_dict_type="core"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]						,     )







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   SudachiTokenizer(normalize_text=UpperCamelCase__						,     sudachi_dict_type="core"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]						,     )







   @require_sudachi
   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   SudachiTokenizer(trim_whitespace=UpperCamelCase__						,     sudachi_dict_type="core"						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]						,     )







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   self.tokenizer_class(self.vocab_file						,     word_tokenizer_type="jumanpp"						)
       self.assertIsNotNone(UpperCamelCase__						)

       snake_case   :				int				 =   """こんにちは、世界。\nこんばんは、世界。"""
       snake_case   :				int				 =   tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

       snake_case   :				Tuple				 =   os.path.join(self.tmpdirname						,     "tokenizer.bin"						)
       with open(UpperCamelCase__						,     "wb"						) as handle:
           pickle.dump(UpperCamelCase__						,     UpperCamelCase__						)

       with open(UpperCamelCase__						,     "rb"						) as handle:
           snake_case   :				Tuple				 =   pickle.load(UpperCamelCase__						)

       snake_case   :				Any				 =   tokenizer_new.tokenize(UpperCamelCase__						)

       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]						,     )







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Any				 =   JumanppTokenizer(do_lower_case=UpperCamelCase__						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]						,     )







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   JumanppTokenizer(normalize_text=UpperCamelCase__						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]						,     )







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   JumanppTokenizer(trim_whitespace=UpperCamelCase__						)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)						,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]						,     )







   @require_jumanpp
   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"						)						,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]						,     )







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       snake_case   :				Any				 =   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

       snake_case   :				str				 =   {}
       for i, token in enumerate(UpperCamelCase__						):
           snake_case   :				Any				 =   i
       snake_case   :				Any				 =   WordpieceTokenizer(vocab=UpperCamelCase__						,     unk_token="[UNK]"						)

       self.assertListEqual(tokenizer.tokenize(""						)						,     []						)

       self.assertListEqual(tokenizer.tokenize("こんにちは"						)						,     ["こんにちは"]						)

       self.assertListEqual(tokenizer.tokenize("こんばんは"						)						,     ["こん", "##ばんは"]						)

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"						)						,     ["こん", "##ばんは", "[UNK]", "こんにちは"]						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				Any				 =   BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"						)
       snake_case   :				str				 =   tokenizer.subword_tokenizer

       snake_case   :				List[Any]				 =   subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"						)
       self.assertListEqual(UpperCamelCase__						,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]						)

       snake_case   :				Tuple				 =   subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"						)
       self.assertListEqual(UpperCamelCase__						,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				Dict				 =   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"						)

       snake_case   :				Tuple				 =   tokenizer.encode("ありがとう。"						,     add_special_tokens=UpperCamelCase__						)
       snake_case   :				Dict				 =   tokenizer.encode("どういたしまして。"						,     add_special_tokens=UpperCamelCase__						)

       snake_case   :				List[str]				 =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						)
       snake_case   :				List[Any]				 =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						,     UpperCamelCase__						)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class _lowerCAmelCase    (				__lowercase   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	List[str]							  =  BertJapaneseTokenizer
   __UpperCAmelCase			:	Union[str, Any]							  =  False







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       super().setUp()

       snake_case   :				List[str]				 =   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       snake_case   :				Optional[Any]				 =   os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["vocab_file"]						)
       with open(self.vocab_file						,     "w"						,     encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname						,     subword_tokenizer_type="character"						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   """こんにちは、世界。 \nこんばんは、世界。"""
       snake_case   :				List[str]				 =   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
       return input_text, output_text







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       pass  # TODO add if relevant







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   self.tokenizer_class(self.vocab_file						,     subword_tokenizer_type="character"						)

       snake_case   :				Tuple				 =   tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"						)
       self.assertListEqual(
           UpperCamelCase__						,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]						)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,     [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)







   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

       snake_case   :				Optional[Any]				 =   {}
       for i, token in enumerate(UpperCamelCase__						):
           snake_case   :				int				 =   i
       snake_case   :				str				 =   CharacterTokenizer(vocab=UpperCamelCase__						,     unk_token="[UNK]"						)

       self.assertListEqual(tokenizer.tokenize(""						)						,     []						)

       self.assertListEqual(tokenizer.tokenize("こんにちは"						)						,     ["こ", "ん", "に", "ち", "は"]						)

       self.assertListEqual(tokenizer.tokenize("こんにちほ"						)						,     ["こ", "ん", "に", "ち", "[UNK]"]						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				int				 =   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"						)

       snake_case   :				List[str]				 =   tokenizer.encode("ありがとう。"						,     add_special_tokens=UpperCamelCase__						)
       snake_case   :				Tuple				 =   tokenizer.encode("どういたしまして。"						,     add_special_tokens=UpperCamelCase__						)

       snake_case   :				Union[str, Any]				 =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						)
       snake_case   :				Any				 =   tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						,     UpperCamelCase__						)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Tuple				 =   """cl-tohoku/bert-base-japanese"""
       snake_case   :				Optional[Any]				 =   AutoTokenizer.from_pretrained(UpperCamelCase__						)
       self.assertIsInstance(UpperCamelCase__						,     UpperCamelCase__						)



class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				str				 =   """cl-tohoku/bert-base-japanese"""
       with self.assertLogs("transformers"						,     level="WARNING"						) as cm:
           BertTokenizer.from_pretrained(UpperCamelCase__						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."						)						)
       snake_case   :				Optional[Any]				 =   """bert-base-cased"""
       with self.assertLogs("transformers"						,     level="WARNING"						) as cm:
           BertJapaneseTokenizer.from_pretrained(UpperCamelCase__						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."						)						)









"""simple docstring"""



import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__snake_case             =   data_utils.TransfoXLTokenizer
__snake_case             =   data_utils.TransfoXLCorpus
__snake_case             =   data_utils
__snake_case             =   data_utils







def   __lowerCAmelCase						(						lowercase  :  Optional[int]						,							lowercase  :  int						,							lowercase  :  List[Any]						,							lowercase  :  Union[str, Any]			)     ->						List[Any]:


    """simple docstring"""





    if transfo_xl_dataset_file:
        # Convert a pre-processed corpus (see original TensorFlow repo)
        with open(lowercase						,							"rb"			) as fp:
            snake_case   :				int				 =   pickle.load(lowercase						,							encoding="latin1"			)
        # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
        snake_case   :				int				 =   pytorch_dump_folder_path + "/" + VOCAB_FILES_NAMES["pretrained_vocab_file"]
        print(F'Save vocabulary to {pytorch_vocab_dump_path}'			)
        snake_case   :				str				 =   corpus.vocab.__dict__
        torch.save(lowercase						,							lowercase			)

        snake_case   :				str				 =   corpus.__dict__
        corpus_dict_no_vocab.pop("vocab"						,							lowercase			)
        snake_case   :				Dict				 =   pytorch_dump_folder_path + "/" + CORPUS_NAME
        print(F'Save dataset to {pytorch_dataset_dump_path}'			)
        torch.save(lowercase						,							lowercase			)

    if tf_checkpoint_path:
        # Convert a pre-trained TensorFlow model
        snake_case   :				Union[str, Any]				 =   os.path.abspath(lowercase			)
        snake_case   :				str				 =   os.path.abspath(lowercase			)

        print(F'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'			)
        # Initialise PyTorch model
        if transfo_xl_config_file == "":
            snake_case   :				int				 =   TransfoXLConfig()
        else:
            snake_case   :				Optional[int]				 =   TransfoXLConfig.from_json_file(lowercase			)
        print(F'Building PyTorch model from configuration: {config}'			)
        snake_case   :				str				 =   TransfoXLLMHeadModel(lowercase			)

        snake_case   :				str				 =   load_tf_weights_in_transfo_xl(lowercase						,							lowercase						,							lowercase			)
        # Save pytorch-model
        snake_case   :				Union[str, Any]				 =   os.path.join(lowercase						,							lowercase			)
        snake_case   :				Optional[Any]				 =   os.path.join(lowercase						,							lowercase			)
        print(F'Save PyTorch model to {os.path.abspath(lowercase			)}'			)
        torch.save(model.state_dict()						,							lowercase			)
        print(F'Save configuration file to {os.path.abspath(lowercase			)}'			)
        with open(lowercase						,							"w"						,							encoding="utf-8"			) as f:
            f.write(config.to_json_string()			)


if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser()
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default=None,
      type=str,
      required=True,
      help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
  )
  parser.add_argument(
      """--tf_checkpoint_path""",
      default="""""",
      type=str,
      help="""An optional path to a TensorFlow checkpoint path to be converted.""",
  )
  parser.add_argument(
      """--transfo_xl_config_file""",
      default="""""",
      type=str,
      help=(
          """An optional config json file corresponding to the pre-trained BERT model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--transfo_xl_dataset_file""",
      default="""""",
      type=str,
      help="""An optional dataset file to be converted in a vocabulary.""",
  )
  __snake_case             =   parser.parse_args()
  convert_transfo_xl_checkpoint_to_pytorch(
      args.tf_checkpoint_path,
      args.transfo_xl_config_file,
      args.pytorch_dump_folder_path,
      args.transfo_xl_dataset_file,
  )





