






import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel


class       __SCREAMING_SNAKE_CASE   :






						def __init__(		self       ,  _UpperCamelCase       ,  _UpperCamelCase=13       ,  _UpperCamelCase=7       ,  _UpperCamelCase=True       ,  _UpperCamelCase=True       ,  _UpperCamelCase=True       ,  _UpperCamelCase=True       ,  _UpperCamelCase=99       ,  _UpperCamelCase=32       ,  _UpperCamelCase=5       ,  _UpperCamelCase=4       ,  _UpperCamelCase=4       ,  _UpperCamelCase="gelu"       ,  _UpperCamelCase=0.0       ,  _UpperCamelCase=0.1       ,  _UpperCamelCase=True       ,  _UpperCamelCase=5_12       ,  _UpperCamelCase=16       ,  _UpperCamelCase=2       ,  _UpperCamelCase=0.02       ,  _UpperCamelCase=3       ,  _UpperCamelCase=4       ,  _UpperCamelCase=None       ,  ):






												"""simple docstring"""
												lowerCAmelCase__        =  parent
												lowerCAmelCase__        =  batch_size
												lowerCAmelCase__        =  seq_length
												lowerCAmelCase__        =  is_training
												lowerCAmelCase__        =  use_input_mask
												lowerCAmelCase__        =  use_token_type_ids
												lowerCAmelCase__        =  use_labels
												lowerCAmelCase__        =  vocab_size
												lowerCAmelCase__        =  hidden_size
												lowerCAmelCase__        =  num_hidden_layers
												lowerCAmelCase__        =  num_attention_heads
												lowerCAmelCase__        =  intermediate_multiple_size
												lowerCAmelCase__        =  hidden_act
												lowerCAmelCase__        =  hidden_dropout
												lowerCAmelCase__        =  attention_dropout
												lowerCAmelCase__        =  weight_tying
												lowerCAmelCase__        =  max_position_embeddings
												lowerCAmelCase__        =  type_vocab_size
												lowerCAmelCase__        =  type_sequence_label_size
												lowerCAmelCase__        =  initializer_range
												lowerCAmelCase__        =  num_labels
												lowerCAmelCase__        =  num_choices
												lowerCAmelCase__        =  scope






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size				)

												lowerCAmelCase__        =  None
												if self.use_input_mask:
																		lowerCAmelCase__        =  random_attention_mask([self.batch_size, self.seq_length]				)

												lowerCAmelCase__        =  None
												if self.use_labels:
																		lowerCAmelCase__        =  ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels				)

												lowerCAmelCase__        =  self.get_config()

												return config, input_ids, input_mask, token_labels






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return GPTNeoXJapaneseConfig(
												    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_multiple_size=self.intermediate_multiple_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout=self.hidden_dropout       ,  attention_dropout=self.attention_dropout       ,  weight_tying=self.weight_tying       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=_UpperCamelCase       ,  initializer_range=self.initializer_range       ,  )






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  self.prepare_config_and_inputs()

												lowerCAmelCase__        =  True

												return config, input_ids, input_mask, token_labels






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  GPTNeoXJapaneseModel(config=_UpperCamelCase				)
												model.to(_UpperCamelCase				)
												model.eval()
												lowerCAmelCase__        =  model(_UpperCamelCase       ,  attention_mask=_UpperCamelCase				)
												lowerCAmelCase__        =  model(_UpperCamelCase				)
												self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  True
												lowerCAmelCase__        =  GPTNeoXJapaneseModel(_UpperCamelCase				)
												model.to(_UpperCamelCase				)
												model.eval()
												lowerCAmelCase__        =  model(_UpperCamelCase       ,  attention_mask=_UpperCamelCase				)
												self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  GPTNeoXJapaneseForCausalLM(config=_UpperCamelCase				)
												model.to(_UpperCamelCase				)
												model.eval()
												lowerCAmelCase__        =  model(_UpperCamelCase       ,  attention_mask=_UpperCamelCase       ,  labels=_UpperCamelCase				)
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  True
												lowerCAmelCase__        =  GPTNeoXJapaneseForCausalLM(config=_UpperCamelCase				)
												model.to(_UpperCamelCase				)
												model.eval()

												# first forward pass
												lowerCAmelCase__        =  model(_UpperCamelCase       ,  attention_mask=_UpperCamelCase       ,  use_cache=_UpperCamelCase				)
												lowerCAmelCase__        =  outputs.past_key_values

												# create hypothetical multiple next token and extent to next_input_ids
												lowerCAmelCase__        =  ids_tensor((self.batch_size, 3)       ,  config.vocab_size				)
												lowerCAmelCase__        =  ids_tensor((self.batch_size, 3)       ,  vocab_size=2				)

												# append to next input_ids and
												lowerCAmelCase__        =  torch.cat([input_ids, next_tokens]       ,  dim=-1				)
												lowerCAmelCase__        =  torch.cat([input_mask, next_mask]       ,  dim=-1				)

												lowerCAmelCase__        =  model(_UpperCamelCase       ,  attention_mask=_UpperCamelCase       ,  output_hidden_states=_UpperCamelCase				)
												lowerCAmelCase__        =  output_from_no_past['hidden_states'][0]
												lowerCAmelCase__        =  model(
												    _UpperCamelCase       ,  attention_mask=_UpperCamelCase       ,  past_key_values=_UpperCamelCase       ,  output_hidden_states=_UpperCamelCase       ,  )['hidden_states'][0]

												# select random slice
												lowerCAmelCase__        =  ids_tensor((1,)       ,  output_from_past.shape[-1]				).item()
												lowerCAmelCase__        =  output_from_no_past[:, -3:, random_slice_idx].detach()
												lowerCAmelCase__        =  output_from_past[:, :, random_slice_idx].detach()

												self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

												# test that outputs are equal for slice
												self.parent.assertTrue(torch.allclose(_UpperCamelCase       ,  _UpperCamelCase       ,  atol=1E-3				)				)







						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  self.prepare_config_and_inputs()
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  config_and_inputs
												lowerCAmelCase__        =  {'input_ids': input_ids, 'attention_mask': input_mask}
												return config, inputs_dict




@require_torch
class       __SCREAMING_SNAKE_CASE   (				__lowercase						,   __lowercase						,   unittest.TestCase):
						_SCREAMING_SNAKE_CASE		:    Dict						=   (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
						_SCREAMING_SNAKE_CASE		:    Optional[int]						=   (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
						_SCREAMING_SNAKE_CASE		:    List[str]						=   (
						    {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
						    if is_torch_available()
						    else {}
						)
						_SCREAMING_SNAKE_CASE		:    Optional[int]						=   False
						_SCREAMING_SNAKE_CASE		:    Dict						=   False
						_SCREAMING_SNAKE_CASE		:    Dict						=   False
						_SCREAMING_SNAKE_CASE		:    Optional[Any]						=   False






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  GPTNeoXJapaneseModelTester(self				)
												lowerCAmelCase__        =  ConfigTester(self       ,  config_class=_UpperCamelCase       ,  hidden_size=37				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												self.config_tester.run_common_tests()






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  self.model_tester.prepare_config_and_inputs_for_decoder()
												self.model_tester.create_and_check_model_as_decoder(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# This regression test was failing with PyTorch < 1.3
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  self.model_tester.prepare_config_and_inputs_for_decoder()

												lowerCAmelCase__        =  None

												self.model_tester.create_and_check_model_as_decoder(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__  ,    lowerCAmelCase__        =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_decoder_model_past_large_inputs(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_causal_lm(*_UpperCamelCase				)







						@slow
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  'abeja/gpt-neox-japanese-2.7b'

												lowerCAmelCase__        =  ['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

												lowerCAmelCase__        =  [
												    'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
												    '100年後に必要とされる会社は、「人」が中心の会社です。',
												    'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
												    '国境の長いトンネルを抜けると、そこは雪国だった。',
												    '美味しい日本食といえば、やっぱりお寿司ですよね。',
												]

												lowerCAmelCase__        =  GPTNeoXJapaneseTokenizer.from_pretrained(_UpperCamelCase				)
												lowerCAmelCase__        =  GPTNeoXJapaneseForCausalLM.from_pretrained(_UpperCamelCase				)

												lowerCAmelCase__        =  []
												for prompt in prompts:
																		lowerCAmelCase__        =  tokenizer(_UpperCamelCase       ,  return_tensors='pt'				).input_ids
																		lowerCAmelCase__        =  model.generate(_UpperCamelCase       ,  max_length=50				)
																		lowerCAmelCase__        =  tokenizer.batch_decode(_UpperCamelCase       ,  skip_special_tokens=_UpperCamelCase				)
																		predicted_outputs += generated_string
												self.assertListEqual(_UpperCamelCase       ,  _UpperCamelCase				)













import unittest

from transformers import BarthezTokenizer, BarthezTokenizerFast, BatchEncoding
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
@require_sentencepiece
@slow  # see https://github.com/huggingface/transformers/issues/11457
class       __SCREAMING_SNAKE_CASE   (				__lowercase						,   unittest.TestCase):
						_SCREAMING_SNAKE_CASE		:    Tuple						=   BarthezTokenizer
						_SCREAMING_SNAKE_CASE		:    int						=   BarthezTokenizerFast
						_SCREAMING_SNAKE_CASE		:    Dict						=   True
						_SCREAMING_SNAKE_CASE		:    Tuple						=   True






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												super().setUp()

												lowerCAmelCase__        =  BarthezTokenizerFast.from_pretrained('moussaKam/mbarthez'				)
												tokenizer.save_pretrained(self.tmpdirname				)
												tokenizer.save_pretrained(self.tmpdirname       ,  legacy_format=_UpperCamelCase				)
												lowerCAmelCase__        =  tokenizer






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  '<pad>'
												lowerCAmelCase__        =  1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase				)       ,  _UpperCamelCase				)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase				)       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  list(self.get_tokenizer().get_vocab().keys()				)

												self.assertEqual(vocab_keys[0]       ,  '<s>'				)
												self.assertEqual(vocab_keys[1]       ,  '<pad>'				)
												self.assertEqual(vocab_keys[-1]       ,  '<mask>'				)
												self.assertEqual(len(_UpperCamelCase				)       ,  10_11_22				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												self.assertEqual(self.get_tokenizer().vocab_size       ,  10_11_22				)






						@require_torch
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												lowerCAmelCase__        =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
												lowerCAmelCase__        =  [0, 57, 30_18, 7_03_07, 91, 2]

												lowerCAmelCase__        =  self.tokenizer(
												    _UpperCamelCase       ,  max_length=len(_UpperCamelCase				)       ,  padding=_UpperCamelCase       ,  truncation=_UpperCamelCase       ,  return_tensors='pt'				)
												self.assertIsInstance(_UpperCamelCase       ,  _UpperCamelCase				)

												self.assertEqual((2, 6)       ,  batch.input_ids.shape				)
												self.assertEqual((2, 6)       ,  batch.attention_mask.shape				)
												lowerCAmelCase__        =  batch.input_ids.tolist()[0]
												self.assertListEqual(_UpperCamelCase       ,  _UpperCamelCase				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												if not self.test_rust_tokenizer:
																		return

												lowerCAmelCase__        =  self.get_tokenizer()
												lowerCAmelCase__        =  self.get_rust_tokenizer()

												lowerCAmelCase__        =  'I was born in 92000, and this is falsé.'

												lowerCAmelCase__        =  tokenizer.tokenize(_UpperCamelCase				)
												lowerCAmelCase__        =  rust_tokenizer.tokenize(_UpperCamelCase				)
												self.assertListEqual(_UpperCamelCase       ,  _UpperCamelCase				)

												lowerCAmelCase__        =  tokenizer.encode(_UpperCamelCase       ,  add_special_tokens=_UpperCamelCase				)
												lowerCAmelCase__        =  rust_tokenizer.encode(_UpperCamelCase       ,  add_special_tokens=_UpperCamelCase				)
												self.assertListEqual(_UpperCamelCase       ,  _UpperCamelCase				)

												lowerCAmelCase__        =  self.get_rust_tokenizer()
												lowerCAmelCase__        =  tokenizer.encode(_UpperCamelCase				)
												lowerCAmelCase__        =  rust_tokenizer.encode(_UpperCamelCase				)
												self.assertListEqual(_UpperCamelCase       ,  _UpperCamelCase				)







						@slow
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												# fmt: off
												lowerCAmelCase__        =  {'input_ids': [[0, 4_90, 1_43_28, 45_07, 3_54, 47, 4_36_69, 95, 25, 7_81_17, 2_02_15, 1_97_79, 1_90, 22, 4_00, 4, 3_53_43, 8_03_10, 6_03, 86, 2_49_37, 1_05, 3_34_38, 9_47_62, 1_96, 3_96_42, 7, 15, 1_59_33, 1_73, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1_05_34, 87, 25, 66, 33_58, 1_96, 5_52_89, 8, 8_29_61, 81, 22_04, 7_52_03, 7, 15, 7_63, 1_29_56, 2_16, 1_78, 1_43_28, 95_95, 13_77, 6_96_93, 7, 4_48, 7_10_21, 1_96, 1_81_06, 14_37, 1_39_74, 1_08, 90_83, 4, 4_93_15, 7, 39, 86, 13_26, 27_93, 4_63_33, 4, 4_48, 1_96, 7_45_88, 7, 4_93_15, 7, 39, 21, 8_22, 3_84_70, 74, 21, 6_67_23, 6_24_80, 8, 2_20_50, 5, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
												# fmt: on

												# moussaKam/mbarthez is a french model. So we also use french texts.
												lowerCAmelCase__        =  [
												    'Le transformeur est un modèle d\'apprentissage profond introduit en 2017, '
												    'utilisé principalement dans le domaine du traitement automatique des langues (TAL).',
												    'À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus '
												    'pour gérer des données séquentielles, telles que le langage naturel, pour des tâches '
												    'telles que la traduction et la synthèse de texte.',
												]

												self.tokenizer_integration_test_util(
												    expected_encoding=_UpperCamelCase       ,  model_name='moussaKam/mbarthez'       ,  revision='c2e4ecbca5e3cd2c37fe1ac285ca4fbdf1366fb6'       ,  sequences=_UpperCamelCase       ,  )





