






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     {'''vocab_file''': '''spiece.model'''}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''bert_for_seq_generation''': (
            '''https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model'''
        ),
    }
}

UpperCamelCase			    =     {'''bert_for_seq_generation''': 512}



class 				UpperCamelCase__		(_UpperCAmelCase   ):
			__snake_case    :				Tuple									=							VOCAB_FILES_NAMES
			__snake_case    :				int									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				str									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				List[int]									=							[]
			__snake_case    :				int									=							["input_ids", "attention_mask"]







			def __init__(     self:  int				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Optional[Any]="<s>"				,     UpperCAmelCase_:  Optional[int]="</s>"				,     UpperCAmelCase_:  Any="<unk>"				,     UpperCAmelCase_:  Any="<pad>"				,     UpperCAmelCase_:  Optional[int]="<::::>"				,     UpperCAmelCase_:  Optional[Dict[str, Any]] = None				,     **UpperCAmelCase_:  Optional[Any]				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {} if sp_model_kwargs is None else sp_model_kwargs

									# Add extra_ids to the special token list
									super().__init__(
									    bos_token=UpperCAmelCase_				,     eos_token=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     sp_model_kwargs=self.sp_model_kwargs				,     **UpperCAmelCase_				,     )

									_SCREAMING_SNAKE_CASE     				=  vocab_file

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(UpperCAmelCase_					)







			@property
			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									return self.sp_model.get_piece_size()







			def 							UpperCamelCase				(     self:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {self.convert_ids_to_tokens(UpperCAmelCase_					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab







			def __getstate__(     self:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.__dict__.copy()
									_SCREAMING_SNAKE_CASE     				=  None
									return state







			def __setstate__(     self:  Any				,     UpperCAmelCase_:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  d

									# for backward compatibility
									if not hasattr(self				,     """sp_model_kwargs"""					):
															_SCREAMING_SNAKE_CASE     				=  {}

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(self.vocab_file					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									return self.sp_model.encode(UpperCAmelCase_				,     out_type=UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									return self.sp_model.piece_to_id(UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.sp_model.IdToPiece(UpperCAmelCase_					)
									return token







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  []
									_SCREAMING_SNAKE_CASE     				=  """"""
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					out_string += self.sp_model.decode(UpperCAmelCase_					) + token
																					_SCREAMING_SNAKE_CASE     				=  []
															else:
																					current_sub_tokens.append(UpperCAmelCase_					)
									out_string += self.sp_model.decode(UpperCAmelCase_					)
									return out_string.strip()







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									if not os.path.isdir(UpperCAmelCase_					):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
															return
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCAmelCase_					) and os.path.isfile(self.vocab_file					):
															copyfile(self.vocab_file				,     UpperCAmelCase_					)
									elif not os.path.isfile(self.vocab_file					):
															with open(UpperCAmelCase_				,     """wb"""					) as fi:
																					_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
																					fi.write(UpperCAmelCase_					)

									return (out_vocab_file,)









import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
		import os

		from flax.core.frozen_dict import unfreeze
		from flax.traverse_util import flatten_dict

		from transformers import FlaxBertModel

		UpperCamelCase			    =     '''0.12'''  # assumed parallelism: 8



@require_flax
@is_staging_test
class 				__UpperCAmelCase		(unittest.TestCase   ):







			@classmethod
			def 							UpperCamelCase				(     cls:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TOKEN
									HfFolder.save_token(UpperCAmelCase_					)







			@classmethod
			def 							UpperCamelCase				(     cls:  Union[str, Any]					):







									'''simple docstring'''




									try:
															delete_repo(token=cls._token				,     repo_id="""test-model-flax"""					)
									except HTTPError:
															pass

									try:
															delete_repo(token=cls._token				,     repo_id="""valid_org/test-model-flax-org"""					)
									except HTTPError:
															pass







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  BertConfig(
									    vocab_size=99				,     hidden_size=32				,     num_hidden_layers=5				,     num_attention_heads=4				,     intermediate_size=37					)
									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel(UpperCAmelCase_					)
									model.push_to_hub("""test-model-flax"""				,     use_auth_token=self._token					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(F'{USER}/test-model-flax'					)

									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(model.params					)					)
									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(new_model.params					)					)

									for key in base_params.keys():
															_SCREAMING_SNAKE_CASE     				=  (base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(UpperCAmelCase_				,     1E-3				,     msg=F'{key} not identical'					)

									# Reset repo
									delete_repo(token=self._token				,     repo_id="""test-model-flax"""					)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(UpperCAmelCase_				,     repo_id="""test-model-flax"""				,     push_to_hub=UpperCAmelCase_				,     use_auth_token=self._token					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(F'{USER}/test-model-flax'					)

									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(model.params					)					)
									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(new_model.params					)					)

									for key in base_params.keys():
															_SCREAMING_SNAKE_CASE     				=  (base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(UpperCAmelCase_				,     1E-3				,     msg=F'{key} not identical'					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  BertConfig(
									    vocab_size=99				,     hidden_size=32				,     num_hidden_layers=5				,     num_attention_heads=4				,     intermediate_size=37					)
									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel(UpperCAmelCase_					)
									model.push_to_hub("""valid_org/test-model-flax-org"""				,     use_auth_token=self._token					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained("""valid_org/test-model-flax-org"""					)

									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(model.params					)					)
									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(new_model.params					)					)

									for key in base_params.keys():
															_SCREAMING_SNAKE_CASE     				=  (base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(UpperCAmelCase_				,     1E-3				,     msg=F'{key} not identical'					)

									# Reset repo
									delete_repo(token=self._token				,     repo_id="""valid_org/test-model-flax-org"""					)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(
															    UpperCAmelCase_				,     repo_id="""valid_org/test-model-flax-org"""				,     push_to_hub=UpperCAmelCase_				,     use_auth_token=self._token					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained("""valid_org/test-model-flax-org"""					)

									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(model.params					)					)
									_SCREAMING_SNAKE_CASE     				=  flatten_dict(unfreeze(new_model.params					)					)

									for key in base_params.keys():
															_SCREAMING_SNAKE_CASE     				=  (base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(UpperCAmelCase_				,     1E-3				,     msg=F'{key} not identical'					)

def 	__lowerCamelCase      (		snake_case__	,snake_case__		)							->  Tuple:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  True
						_SCREAMING_SNAKE_CASE     				=  flatten_dict(modela.params		)
						_SCREAMING_SNAKE_CASE     				=  flatten_dict(modela.params		)
						for key in flat_params_a.keys():
												if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]		)		) > 1e-4:
																		_SCREAMING_SNAKE_CASE     				=  False

						return models_are_equal




@require_flax
class 				__UpperCAmelCase		(unittest.TestCase   ):







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  BertConfig.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""					)
									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  """bert"""
									with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(os.path.join(UpperCAmelCase_				,     UpperCAmelCase_					)					)

															with self.assertRaises(UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_					)

															_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_				,     subfolder=UpperCAmelCase_					)

									self.assertTrue(check_models_equal(UpperCAmelCase_				,     UpperCAmelCase_					)					)







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  BertConfig.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""					)
									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  """bert"""
									with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(os.path.join(UpperCAmelCase_				,     UpperCAmelCase_					)				,     max_shard_size="""10KB"""					)

															with self.assertRaises(UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_					)

															_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_				,     subfolder=UpperCAmelCase_					)

									self.assertTrue(check_models_equal(UpperCAmelCase_				,     UpperCAmelCase_					)					)







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """bert"""
									_SCREAMING_SNAKE_CASE     				=  """hf-internal-testing/tiny-random-bert-subfolder"""

									with self.assertRaises(UpperCAmelCase_					):
															_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_				,     subfolder=UpperCAmelCase_					)

									self.assertIsNotNone(UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """bert"""
									_SCREAMING_SNAKE_CASE     				=  """hf-internal-testing/tiny-random-bert-sharded-subfolder"""
									with self.assertRaises(UpperCAmelCase_					):
															_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  FlaxBertModel.from_pretrained(UpperCAmelCase_				,     subfolder=UpperCAmelCase_					)

									self.assertIsNotNone(UpperCAmelCase_					)

