

# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
					import jax
					import jaxlib

a		=get_logger()

a		=None






class 	A_		(	TensorFormatter[Mapping, '''jax.Array''', Mapping]    ):
							def __init__(		self		:      Tuple       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]=None       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]=None       ,**SCREAMING_SNAKE_CASE__		:      int):
								super().__init__(features=SCREAMING_SNAKE_CASE__)
								import jax
								from jaxlib.xla_client import Device

								if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__):
									raise ValueError(
									    F"Expected {device} to be a `str` not {type(SCREAMING_SNAKE_CASE__)}, as `jaxlib.xla_extension.Device` "
									    'is not serializable neither with `pickle` nor with `dill`. Instead you can surround '
									    'the device with `str()` to get its string identifier that will be internally mapped '
									    'to the actual `jaxlib.xla_extension.Device`.')
								__lowerCamelCase    :    int 	=			device if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__) else str(jax.devices()[0])
								# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
								# with `pickle` nor with `dill`, so we need to use a global variable instead
								global DEVICE_MAPPING
								if DEVICE_MAPPING is None:
									__lowerCamelCase    :    Any 	=			self._map_devices_to_str()
								if self.device not in list(DEVICE_MAPPING.keys()):
									logger.warning(
									    F"Device with string identifier {self.device} not listed among the available "
									    F"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default "
									    F"device: {str(jax.devices()[0])}.")
									__lowerCamelCase    :    Union[str, Any] 	=			str(jax.devices()[0])
								__lowerCamelCase    :    List[str] 	=			jnp_array_kwargs
							@staticmethod
							def    lowerCAmelCase					(		):
								import jax

								return {str(SCREAMING_SNAKE_CASE__): device for device in jax.devices()}
							def    lowerCAmelCase					(		self		:      Optional[int]       ,SCREAMING_SNAKE_CASE__		:      Any):
								import jax
								import jax.numpy as jnp

								if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__) and column:
									if all(
									    isinstance(SCREAMING_SNAKE_CASE__       ,jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column):
										return jnp.stack(SCREAMING_SNAKE_CASE__       ,axis=0)
								return column
							def    lowerCAmelCase					(		self		:      int       ,SCREAMING_SNAKE_CASE__		:      Optional[Any]):
								import jax
								import jax.numpy as jnp

								if isinstance(SCREAMING_SNAKE_CASE__       ,(str, bytes, type(SCREAMING_SNAKE_CASE__))):
									return value
								elif isinstance(SCREAMING_SNAKE_CASE__       ,(np.character, np.ndarray)) and np.issubdtype(value.dtype       ,np.character):
									return value.tolist()

								__lowerCamelCase    :    Optional[int] 	=			{}

								if isinstance(SCREAMING_SNAKE_CASE__       ,(np.number, np.ndarray)) and np.issubdtype(value.dtype       ,np.integer):
									# the default int precision depends on the jax config
									# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
									if jax.config.jax_enable_xaa:
										__lowerCamelCase    :    List[Any] 	=			{'dtype': jnp.intaa}
									else:
										__lowerCamelCase    :    Optional[int] 	=			{'dtype': jnp.intaa}
								elif isinstance(SCREAMING_SNAKE_CASE__       ,(np.number, np.ndarray)) and np.issubdtype(value.dtype       ,np.floating):
									__lowerCamelCase    :    Tuple 	=			{'dtype': jnp.floataa}
								elif config.PIL_AVAILABLE and "PIL" in sys.modules:
									import PIL.Image

									if isinstance(SCREAMING_SNAKE_CASE__       ,PIL.Image.Image):
										__lowerCamelCase    :    List[Any] 	=			np.asarray(SCREAMING_SNAKE_CASE__)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
								global DEVICE_MAPPING
								if DEVICE_MAPPING is None:
									__lowerCamelCase    :    Dict 	=			self._map_devices_to_str()

								with jax.default_device(DEVICE_MAPPING[self.device]):
									# calling jnp.array on a np.ndarray does copy the data
									# see https://github.com/google/jax/issues/4486
									return jnp.array(SCREAMING_SNAKE_CASE__       ,**{**default_dtype, **self.jnp_array_kwargs})
							def    lowerCAmelCase					(		self		:      Optional[Any]       ,SCREAMING_SNAKE_CASE__		:      Dict):
								import jax

								# support for torch, tf, jax etc.
								if config.TORCH_AVAILABLE and "torch" in sys.modules:
									import torch

									if isinstance(SCREAMING_SNAKE_CASE__       ,torch.Tensor):
										return self._tensorize(data_struct.detach().cpu().numpy()[()])
								if hasattr(SCREAMING_SNAKE_CASE__       ,'__array__') and not isinstance(SCREAMING_SNAKE_CASE__       ,jax.Array):
									__lowerCamelCase    :    int 	=			data_struct.__array__()
								# support for nested types like struct of list of struct
								if isinstance(SCREAMING_SNAKE_CASE__       ,np.ndarray):
									if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
										return self._consolidate([self.recursive_tensorize(SCREAMING_SNAKE_CASE__) for substruct in data_struct])
								elif isinstance(SCREAMING_SNAKE_CASE__       ,(list, tuple)):
									return self._consolidate([self.recursive_tensorize(SCREAMING_SNAKE_CASE__) for substruct in data_struct])
								return self._tensorize(SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      Any       ,SCREAMING_SNAKE_CASE__		:      dict):
								return map_nested(self._recursive_tensorize       ,SCREAMING_SNAKE_CASE__       ,map_list=SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      Tuple       ,SCREAMING_SNAKE_CASE__		:      pa.Table):
								__lowerCamelCase    :    List[str] 	=			self.numpy_arrow_extractor().extract_row(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    List[str] 	=			self.python_features_decoder.decode_row(SCREAMING_SNAKE_CASE__)
								return self.recursive_tensorize(SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      str       ,SCREAMING_SNAKE_CASE__		:      pa.Table):
								__lowerCamelCase    :    Any 	=			self.numpy_arrow_extractor().extract_column(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Any 	=			self.python_features_decoder.decode_column(SCREAMING_SNAKE_CASE__       ,pa_table.column_names[0])
								__lowerCamelCase    :    List[str] 	=			self.recursive_tensorize(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Any 	=			self._consolidate(SCREAMING_SNAKE_CASE__)
								return column


							def    lowerCAmelCase					(		self		:      Union[str, Any]       ,SCREAMING_SNAKE_CASE__		:      pa.Table):
								__lowerCamelCase    :    Union[str, Any] 	=			self.numpy_arrow_extractor().extract_batch(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Any 	=			self.python_features_decoder.decode_batch(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase    :    Optional[int] 	=			self.recursive_tensorize(SCREAMING_SNAKE_CASE__)
								for column_name in batch:
									__lowerCamelCase    :    int 	=			self._consolidate(batch[column_name])
								return batch







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    List[Any] 	=			mock.Mock()
								__lowerCamelCase    :    Tuple 	=			5_0_0
								__lowerCamelCase    :    Dict 	=			{}
								__lowerCamelCase    :    List[Any] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Optional[int] 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Tuple 	=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
									# This check we did call the fake head request
									mock_head.assert_called()
							@require_tokenizers
							def    lowerCAmelCase					(		self		:      Any):
								# A mock response for an HTTP head request to emulate server down
								__lowerCamelCase    :    int 	=			mock.Mock()
								__lowerCamelCase    :    List[str] 	=			5_0_0
								__lowerCamelCase    :    Tuple 	=			{}
								__lowerCamelCase    :    List[str] 	=			HTTPError
								__lowerCamelCase    :    int 	=			{}

								# Download this model to make sure it's in the cache.
								__lowerCamelCase    :    Dict 	=			GPTaTokenizerFast.from_pretrained('gpt2')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'       ,return_value=SCREAMING_SNAKE_CASE__) as mock_head:
									__lowerCamelCase    :    Optional[int] 	=			GPTaTokenizerFast.from_pretrained('gpt2')
									# This check we did call the fake head request
									mock_head.assert_called()
							def    lowerCAmelCase					(		self		:      str):
								# This test is for deprecated behavior and can be removed in v5
								try:
									__lowerCamelCase    :    Tuple 	=			tempfile.mktemp()
									with open(SCREAMING_SNAKE_CASE__       ,'wb') as f:
										http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Dict 	=			AlbertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__)
								finally:
									os.remove(SCREAMING_SNAKE_CASE__)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('tokenizer.json'):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
								try:
									with open('tokenizer.json'       ,'wb') as f:
										http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    Any 	=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size       ,1_0_0_0)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
									os.remove('tokenizer.json')


							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# This test is for deprecated behavior and can be removed in v5
								__lowerCamelCase    :    str 	=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')






@is_staging_test
class 	A_		(	unittest.TestCase    ):
							_UpperCAmelCase   :  Any										=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']
							@classmethod
							def    lowerCAmelCase					(		cls		:      Optional[int]):
								__lowerCamelCase    :    Optional[int] 	=			TOKEN
								HfFolder.save_token(SCREAMING_SNAKE_CASE__)
							@classmethod
							def    lowerCAmelCase					(		cls		:      str):
								try:
									delete_repo(token=cls._token       ,repo_id='test-tokenizer')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='valid_org/test-tokenizer-org')
								except HTTPError:
									pass

								try:
									delete_repo(token=cls._token       ,repo_id='test-dynamic-tokenizer')
								except HTTPError:
									pass
							def    lowerCAmelCase					(		self		:      List[str]):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Union[str, Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-tokenizer'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[str] 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='test-tokenizer')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__       ,repo_id='test-tokenizer'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    str 	=			BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)
							def    lowerCAmelCase					(		self		:      Any):
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Optional[Any] 	=			BertTokenizer(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,use_auth_token=self._token)
								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token       ,repo_id='valid_org/test-tokenizer-org')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    SCREAMING_SNAKE_CASE__       ,repo_id='valid_org/test-tokenizer-org'       ,push_to_hub=SCREAMING_SNAKE_CASE__       ,use_auth_token=self._token)

								__lowerCamelCase    :    List[Any] 	=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab)


							@require_tokenizers
							def    lowerCAmelCase					(		self		:      int):
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    Dict 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									__lowerCamelCase    :    Any 	=			CustomTokenizer(SCREAMING_SNAKE_CASE__)

								# No fast custom tokenizer
								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
									__lowerCamelCase    :    List[str] 	=			os.path.join(SCREAMING_SNAKE_CASE__       ,'vocab.txt')
									with open(SCREAMING_SNAKE_CASE__       ,'w'       ,encoding='utf-8') as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

									__lowerCamelCase    :    Dict 	=			BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)
									bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__)
									__lowerCamelCase    :    List[Any] 	=			CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__)

								tokenizer.push_to_hub('test-dynamic-tokenizer'       ,use_auth_token=self._token)

								__lowerCamelCase    :    Optional[Any] 	=			AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizerFast')
								__lowerCamelCase    :    Optional[int] 	=			AutoTokenizer.from_pretrained(
								    F"{USER}/test-dynamic-tokenizer"       ,use_fast=SCREAMING_SNAKE_CASE__       ,trust_remote_code=SCREAMING_SNAKE_CASE__)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,'CustomTokenizer')






class 	A_		(	unittest.TestCase    ):
							def    lowerCAmelCase					(		self		:      List[Any]):
								__lowerCamelCase    :    List[str] 	=			Trie()
								trie.add('Hello 友達')
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
								trie.add('Hello')
								trie.data
								self.assertEqual(trie.data       ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})
							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    str 	=			Trie()
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS] This is a extra_id_100'])
								trie.add('[CLS]')
								trie.add('extra_id_1')
								trie.add('extra_id_100')
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')       ,['[CLS]', ' This is a ', 'extra_id_100'])
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('A')
								self.assertEqual(trie.split('ABC')       ,['A', 'BC'])
								self.assertEqual(trie.split('BCA')       ,['BC', 'A'])
							def    lowerCAmelCase					(		self		:      Dict):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('TOKEN]')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      int):
								__lowerCamelCase    :    List[Any] 	=			Trie()
								trie.add('A')
								trie.add('P')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')       ,['This is something ', '[SPECIAL_TOKEN]'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Any 	=			Trie()
								trie.add('AB')
								trie.add('B')
								trie.add('C')
								self.assertEqual(trie.split('ABC')       ,['AB', 'C'])
							def    lowerCAmelCase					(		self		:      Optional[int]):
								__lowerCamelCase    :    Dict 	=			Trie()
								trie.add('ABC')
								trie.add('B')
								trie.add('CD')
								self.assertEqual(trie.split('ABCD')       ,['ABC', 'D'])


							def    lowerCAmelCase					(		self		:      List[Any]):
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								__lowerCamelCase    :    List[Any] 	=			Trie()
								__lowerCamelCase    :    Any 	=			trie.cut_text('ABC'       ,[0, 0, 2, 1, 2, 3])
								self.assertEqual(SCREAMING_SNAKE_CASE__       ,['AB', 'C'])




