import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					A							(    UpperCAmelCase_		,    unittest.TestCase					):
   __UpperCAmelCase			:			Union[str, Any]              =      GPTSanJapaneseTokenizer
   __UpperCAmelCase			:			Dict              =      False
   __UpperCAmelCase			:			List[str]              =      {'do_clean_text': False, 'add_prefix_space': False}


   def       lowercase_				(self					:						Dict		)						->    Union[str, Any]:
         """simple docstring"""



         super().setUp()

         # fmt: off
         UpperCAmelCase__								=  ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
         # fmt: on
         UpperCAmelCase__								=  {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
         UpperCAmelCase__								=  {"unk_token": "<unk>"}

         UpperCAmelCase__								=  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]		)
         UpperCAmelCase__								=  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["emoji_file"]		)
         with open(self.vocab_file	,  "w"	,  encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
         with open(self.emoji_file	,  "w"		) as emoji_writer:
               emoji_writer.write(json.dumps(__UpperCAmelCase		)		)


   def       lowercase_				(self					:						List[str]	,  **__UpperCAmelCase					:						Union[str, Any]		)						->    List[str]:
         """simple docstring"""



         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname	,  **__UpperCAmelCase		)


   def       lowercase_				(self					:						Dict	,  __UpperCAmelCase					:						Any		)						->    Union[str, Any]:
         """simple docstring"""



         UpperCAmelCase__								=  "こんにちは、世界。 \nこんばんは、㔺界。😀"
         UpperCAmelCase__								=  "こんにちは、世界。 \nこんばんは、世界。😀"
         return input_text, output_text


   def       lowercase_				(self					:						str	,  __UpperCAmelCase					:						Tuple		)						->    List[Any]:
         """simple docstring"""



         UpperCAmelCase__		,				UpperCAmelCase__								=  self.get_input_output_texts(__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.encode(__UpperCAmelCase	,  add_special_tokens=__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.decode(__UpperCAmelCase	,  clean_up_tokenization_spaces=__UpperCAmelCase		)
         return text, ids


   def       lowercase_				(self					:						Union[str, Any]		)						->    List[Any]:
         """simple docstring"""



         pass  # TODO add if relevant


   def       lowercase_				(self					:						Optional[Any]		)						->    int:
         """simple docstring"""



         pass  # TODO add if relevant


   def       lowercase_				(self					:						List[Any]		)						->    Optional[Any]:
         """simple docstring"""



         pass  # TODO add if relevant


   def       lowercase_				(self					:						List[str]		)						->    str:
         """simple docstring"""



         UpperCAmelCase__								=  self.get_tokenizer()

         # Testing tokenization
         UpperCAmelCase__								=  "こんにちは、世界。　こんばんは、㔺界。"
         UpperCAmelCase__								=  ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
         UpperCAmelCase__								=  tokenizer.tokenize(__UpperCAmelCase		)
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)

         # Testing conversion to ids without special tokens
         UpperCAmelCase__								=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         UpperCAmelCase__								=  tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)

         # Testing conversion to ids with special tokens
         UpperCAmelCase__								=  tokens + [tokenizer.unk_token]
         UpperCAmelCase__								=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
         UpperCAmelCase__								=  tokenizer.convert_tokens_to_ids(__UpperCAmelCase		)
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)


   def       lowercase_				(self					:						List[str]		)						->    List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.get_tokenizer()

         # Testing tokenization
         UpperCAmelCase__								=  "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
         UpperCAmelCase__								=  "こんにちは、、、、世界。こんばんは、、、、世界。"
         UpperCAmelCase__								=  tokenizer.encode(__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.decode(__UpperCAmelCase		)
         self.assertEqual(__UpperCAmelCase	,  __UpperCAmelCase		)


   @slow
   def       lowercase_				(self					:						str		)						->    Tuple:
         """simple docstring"""



         UpperCAmelCase__								=  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

         # Testing tokenization
         UpperCAmelCase__								=  "こんにちは、世界。"
         UpperCAmelCase__								=  "こんばんは、㔺界。😀"
         UpperCAmelCase__								=  "こんにちは、世界。こんばんは、世界。😀"
         UpperCAmelCase__								=  tokenizer.encode(prefix_text + input_text		)
         UpperCAmelCase__								=  tokenizer.encode(""	,  prefix_text=prefix_text + input_text		)
         UpperCAmelCase__								=  tokenizer.encode(__UpperCAmelCase	,  prefix_text=__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.decode(__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.decode(__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.decode(__UpperCAmelCase		)
         self.assertEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertEqual(__UpperCAmelCase	,  __UpperCAmelCase		)


   @slow
   def       lowercase_				(self					:						int		)						->    Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

         # Testing tokenization
         UpperCAmelCase__								=  "こんにちは、世界。"
         UpperCAmelCase__								=  "こんばんは、㔺界。😀"

         UpperCAmelCase__								=  len(tokenizer.encode(__UpperCAmelCase		)		) - 2
         UpperCAmelCase__								=  len(tokenizer.encode(__UpperCAmelCase		)		) - 2

         UpperCAmelCase__								=  [1] + [0] * (len_prefix + len_text + 1)
         UpperCAmelCase__								=  [1] * (len_prefix + len_text + 1) + [0]
         UpperCAmelCase__								=  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

         UpperCAmelCase__								=  tokenizer(prefix_text + input_text		).token_type_ids
         UpperCAmelCase__								=  tokenizer(""	,  prefix_text=prefix_text + input_text		).token_type_ids
         UpperCAmelCase__								=  tokenizer(__UpperCAmelCase	,  prefix_text=__UpperCAmelCase		).token_type_ids
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertListEqual(__UpperCAmelCase	,  __UpperCAmelCase		)


   @slow
   def       lowercase_				(self					:						Optional[int]		)						->    Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

         UpperCAmelCase__								=  tokenizer.encode("あンいワ"		)
         UpperCAmelCase__								=  tokenizer.encode(""	,  prefix_text="あンいワ"		)
         UpperCAmelCase__								=  tokenizer.encode("いワ"	,  prefix_text="あン"		)

         self.assertEqual(tokenizer.decode(__UpperCAmelCase		)	,  tokenizer.decode(__UpperCAmelCase		)		)
         self.assertEqual(tokenizer.decode(__UpperCAmelCase		)	,  tokenizer.decode(__UpperCAmelCase		)		)
         self.assertNotEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertNotEqual(__UpperCAmelCase	,  __UpperCAmelCase		)
         self.assertEqual(x_token_a[1]	,  x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]	,  x_token_a[3]		)  # SEG token


   @slow
   def       lowercase_				(self					:						int		)						->    int:
         """simple docstring"""



         UpperCAmelCase__								=  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

         UpperCAmelCase__								=  [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
         UpperCAmelCase__								=  tokenizer(__UpperCAmelCase	,  padding=__UpperCAmelCase		)
         UpperCAmelCase__								=  tokenizer.batch_encode_plus(__UpperCAmelCase	,  padding=__UpperCAmelCase		)

         # fmt: off
         UpperCAmelCase__								=  [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
         UpperCAmelCase__								=  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         UpperCAmelCase__								=  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids	,  __UpperCAmelCase		)
         self.assertListEqual(x_token.token_type_ids	,  __UpperCAmelCase		)
         self.assertListEqual(x_token.attention_mask	,  __UpperCAmelCase		)
         self.assertListEqual(x_token_a.input_ids	,  __UpperCAmelCase		)
         self.assertListEqual(x_token_a.token_type_ids	,  __UpperCAmelCase		)
         self.assertListEqual(x_token_a.attention_mask	,  __UpperCAmelCase		)


   def       lowercase_				(self					:						Optional[int]		)						->    Any:
         """simple docstring"""



         pass





   def       lowercase_				(self					:						List[str]		)						->    str:
         """simple docstring"""



         pass




from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__    							= logging.get_logger(__name__)

UpperCamelCase__    							= {
    'microsoft/xprophetnet-large-wiki100-cased': (
        'https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/config.json'
    ),
}





class 					A							(    UpperCAmelCase_					):
   __UpperCAmelCase			:			Optional[int]              =      'xlm-prophetnet'
   __UpperCAmelCase			:			int              =      ['past_key_values']
   __UpperCAmelCase			:			Optional[int]              =      {
       'num_attention_heads': 'num_encoder_attention_heads',
   }


   def __init__(self					:						str	,  __UpperCAmelCase					:						Optional[float] = 0.1	,  __UpperCAmelCase					:						Optional[Union[str, Callable]] = "gelu"	,  __UpperCAmelCase					:						Optional[int] = 3_0_5_2_2	,  __UpperCAmelCase					:						Optional[int] = 1_0_2_4	,  __UpperCAmelCase					:						Optional[int] = 4_0_9_6	,  __UpperCAmelCase					:						Optional[int] = 1_2	,  __UpperCAmelCase					:						Optional[int] = 1_6	,  __UpperCAmelCase					:						Optional[int] = 4_0_9_6	,  __UpperCAmelCase					:						Optional[int] = 1_2	,  __UpperCAmelCase					:						Optional[int] = 1_6	,  __UpperCAmelCase					:						Optional[float] = 0.1	,  __UpperCAmelCase					:						Optional[float] = 0.1	,  __UpperCAmelCase					:						Optional[int] = 5_1_2	,  __UpperCAmelCase					:						Optional[float] = 0.02	,  __UpperCAmelCase					:						Optional[bool] = True	,  __UpperCAmelCase					:						Optional[bool] = True	,  __UpperCAmelCase					:						Optional[int] = 0	,  __UpperCAmelCase					:						Optional[int] = 2	,  __UpperCAmelCase					:						Optional[int] = 3_2	,  __UpperCAmelCase					:						Optional[int] = 1_2_8	,  __UpperCAmelCase					:						Optional[bool] = False	,  __UpperCAmelCase					:						Optional[float] = 0.0	,  __UpperCAmelCase					:						Optional[bool] = True	,  __UpperCAmelCase					:						Optional[int] = 0	,  __UpperCAmelCase					:						Optional[int] = 1	,  __UpperCAmelCase					:						Optional[int] = 2	,  **__UpperCAmelCase					:						Optional[Any]	,  )						->    List[str]:
         """simple docstring"""



         UpperCAmelCase__								=  vocab_size
         UpperCAmelCase__								=  hidden_size
         UpperCAmelCase__								=  encoder_ffn_dim
         UpperCAmelCase__								=  num_encoder_layers
         UpperCAmelCase__								=  num_encoder_attention_heads
         UpperCAmelCase__								=  decoder_ffn_dim
         UpperCAmelCase__								=  num_decoder_layers
         UpperCAmelCase__								=  num_decoder_attention_heads
         UpperCAmelCase__								=  max_position_embeddings
         UpperCAmelCase__								=  init_std  # Normal(0, this parameter)
         UpperCAmelCase__								=  activation_function

         # parameters for xlmprophetnet
         UpperCAmelCase__								=  ngram
         UpperCAmelCase__								=  num_buckets
         UpperCAmelCase__								=  relative_max_distance
         UpperCAmelCase__								=  disable_ngram_loss
         UpperCAmelCase__								=  eps

         # 3 Types of Dropout
         UpperCAmelCase__								=  attention_dropout
         UpperCAmelCase__								=  activation_dropout
         UpperCAmelCase__								=  dropout

         UpperCAmelCase__								=  use_cache

         super().__init__(
             pad_token_id=__UpperCAmelCase	,  bos_token_id=__UpperCAmelCase	,  eos_token_id=__UpperCAmelCase	,  is_encoder_decoder=__UpperCAmelCase	,  add_cross_attention=__UpperCAmelCase	,  decoder_start_token_id=__UpperCAmelCase	,  **__UpperCAmelCase	,  )


   @property
   def       lowercase_				(self					:						int		)						->    int:
         """simple docstring"""



         return self.num_encoder_layers + self.num_decoder_layers





   @num_hidden_layers.setter
   def       lowercase_				(self					:						Any	,  __UpperCAmelCase					:						List[Any]		)						->    Union[str, Any]:
         """simple docstring"""



         raise NotImplementedError(
             "This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and"
             " `num_decoder_layers`."		)



