


import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 	snake_case_       (__lowercase     ,						unittest.TestCase				):
					UpperCAmelCase__ :   List[Any]   				=      LongformerTokenizer
					UpperCAmelCase__ :   Optional[int]   				=      True
					UpperCAmelCase__ :   List[str]   				=      LongformerTokenizerFast
					UpperCAmelCase__ :   Any   				=      True
					def 			lowerCamelCase__(				self       :Any				) ->   int:
												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												a__					=					[
												    '''l''',
												    '''o''',
												    '''w''',
												    '''e''',
												    '''r''',
												    '''s''',
												    '''t''',
												    '''i''',
												    '''d''',
												    '''n''',
												    '''\u0120''',
												    '''\u0120l''',
												    '''\u0120n''',
												    '''\u0120lo''',
												    '''\u0120low''',
												    '''er''',
												    '''\u0120lowest''',
												    '''\u0120newer''',
												    '''\u0120wider''',
												    '''<unk>''',
												]
												a__					=					dict(zip(_a      ,range(len(_a				)				)				)				)
												a__					=					['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
												a__					=					{'''unk_token''': '''<unk>'''}

												a__					=					os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['vocab_file']				)
												a__					=					os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['merges_file']				)
												with open(self.vocab_file      ,'w'      ,encoding='utf-8'				) as fp:
																			fp.write(json.dumps(_a				) + '\n'				)
												with open(self.merges_file      ,'w'      ,encoding='utf-8'				) as fp:
																			fp.write('\n'.join(_a				)				)
					def 			lowerCamelCase__(				self       :Union[str, Any]      ,**__snake_case       :Union[str, Any]				) ->   int:
												kwargs.update(self.special_tokens_map				)
												return self.tokenizer_class.from_pretrained(self.tmpdirname      ,**_a				)
					def 			lowerCamelCase__(				self       :Tuple      ,**__snake_case       :Optional[int]				) ->   Tuple:
												kwargs.update(self.special_tokens_map				)
												return self.rust_tokenizer_class.from_pretrained(self.tmpdirname      ,**_a				)
					def 			lowerCamelCase__(				self       :List[str]      ,__snake_case       :Dict				) ->   Union[str, Any]:
												a__					=					'''lower newer'''
												a__					=					'''lower newer'''
												return input_text, output_text
					def 			lowerCamelCase__(				self       :Optional[int]				) ->   Union[str, Any]:
												a__					=					self.tokenizer_class(self.vocab_file      ,self.merges_file      ,**self.special_tokens_map				)
												a__					=					'''lower newer'''
												a__					=					['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
												a__					=					tokenizer.tokenize(_a				)  # , add_prefix_space=True)
												self.assertListEqual(_a      ,_a				)

												a__					=					tokens + [tokenizer.unk_token]
												a__					=					[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)      ,_a				)
					def 			lowerCamelCase__(				self       :Dict				) ->   int:
												a__					=					self.get_tokenizer()

												self.assertListEqual(tokenizer.encode('Hello world!'      ,add_special_tokens=_a				)      ,[0, 3_14_14, 2_32, 3_28, 2]				)
												self.assertListEqual(
												    tokenizer.encode('Hello world! cécé herlolip 418'      ,add_special_tokens=_a				)      ,[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]      ,)
					@slow
					def 			lowerCamelCase__(				self       :Optional[int]				) ->   List[str]:
												a__					=					self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'				)

												a__					=					tokenizer.encode('sequence builders'      ,add_special_tokens=_a				)
												a__					=					tokenizer.encode('multi-sequence build'      ,add_special_tokens=_a				)

												a__					=					tokenizer.encode(
												    'sequence builders'      ,add_special_tokens=_a      ,add_prefix_space=_a				)
												a__					=					tokenizer.encode(
												    'sequence builders'      ,'multi-sequence build'      ,add_special_tokens=_a      ,add_prefix_space=_a				)

												a__					=					tokenizer.build_inputs_with_special_tokens(_a				)
												a__					=					tokenizer.build_inputs_with_special_tokens(_a      ,_a				)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode
					def 			lowerCamelCase__(				self       :List[Any]				) ->   Optional[int]:
												a__					=					self.get_tokenizer()

												a__					=					'''Encode this sequence.'''
												a__					=					tokenizer.byte_encoder[''' '''.encode('utf-8'				)[0]]

												# Testing encoder arguments
												a__					=					tokenizer.encode(_a      ,add_special_tokens=_a      ,add_prefix_space=_a				)
												a__					=					tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
												self.assertNotEqual(_a      ,_a				)

												a__					=					tokenizer.encode(_a      ,add_special_tokens=_a      ,add_prefix_space=_a				)
												a__					=					tokenizer.convert_ids_to_tokens(encoded[0]				)[0]
												self.assertEqual(_a      ,_a				)

												tokenizer.add_special_tokens({'bos_token': '<s>'}				)
												a__					=					tokenizer.encode(_a      ,add_special_tokens=_a				)
												a__					=					tokenizer.convert_ids_to_tokens(encoded[1]				)[0]
												self.assertNotEqual(_a      ,_a				)

												# Testing spaces after special tokens
												a__					=					'''<mask>'''
												tokenizer.add_special_tokens(
												    {'mask_token': AddedToken(_a      ,lstrip=_a      ,rstrip=_a				)}				)  # mask token has a left space
												a__					=					tokenizer.convert_tokens_to_ids(_a				)

												a__					=					'''Encode <mask> sequence'''
												a__					=					'''Encode <mask>sequence'''

												a__					=					tokenizer.encode(_a				)
												a__					=					encoded.index(_a				)
												a__					=					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
												self.assertEqual(_a      ,_a				)

												a__					=					tokenizer.encode(_a				)
												a__					=					encoded.index(_a				)
												a__					=					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]				)[0]
												self.assertNotEqual(_a      ,_a				)
					def 			lowerCamelCase__(				self       :Optional[Any]				) ->   Any:
												pass
					def 			lowerCamelCase__(				self       :int				) ->   str:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																										a__					=					self.rust_tokenizer_class.from_pretrained(_a      ,**_a				)
																										a__					=					self.tokenizer_class.from_pretrained(_a      ,**_a				)
																										a__					=					'''A, <mask> AllenNLP sentence.'''
																										a__					=					tokenizer_r.encode_plus(_a      ,add_special_tokens=_a      ,return_token_type_ids=_a				)
																										a__					=					tokenizer_p.encode_plus(_a      ,add_special_tokens=_a      ,return_token_type_ids=_a				)

																										# token_type_ids should put 0 everywhere
																										self.assertEqual(sum(tokens_r['token_type_ids']				)      ,sum(tokens_p['token_type_ids']				)				)

																										# attention_mask should put 1 everywhere, so sum over length should be 1
																										self.assertEqual(
																										    sum(tokens_r['attention_mask']				) / len(tokens_r['attention_mask']				)      ,sum(tokens_p['attention_mask']				) / len(tokens_p['attention_mask']				)      ,)

																										a__					=					tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']				)
																										a__					=					tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']				)

																										# Rust correctly handles the space before the mask while python doesnt
																										self.assertSequenceEqual(tokens_p['input_ids']      ,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)
																										self.assertSequenceEqual(tokens_r['input_ids']      ,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)

																										self.assertSequenceEqual(
																										    _a      ,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
																										self.assertSequenceEqual(
																										    _a      ,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
					def 			lowerCamelCase__(				self       :Dict				) ->   Optional[int]:
												for trim_offsets, add_prefix_space in itertools.product([True, False]      ,repeat=2				):
																			a__					=					self.rust_tokenizer_class.from_pretrained(
																			    self.tmpdirname      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)

																			a__					=					json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()				)
																			a__					=					json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()				)

																			self.assertEqual(pre_tokenizer_state['add_prefix_space']      ,_a				)

																			self.assertEqual(post_processor_state['add_prefix_space']      ,_a				)
																			self.assertEqual(post_processor_state['trim_offsets']      ,_a				)



					def 			lowerCamelCase__(				self       :Tuple				) ->   Union[str, Any]:
												# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
												# `trim_offsets`
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																										a__					=					'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																										a__					=					F'{text_of_1_token} {text_of_1_token}'

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(len(_a				) + 1, len(_a				) + 1 + len(_a				))      ,)

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(len(_a				) + 1, len(_a				) + 1 + len(_a				))      ,)

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(len(_a				), len(_a				) + 1 + len(_a				))      ,)

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(len(_a				), len(_a				) + 1 + len(_a				))      ,)

																										a__					=					F' {text}'

																										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																										# )
																										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																										# self.assertEqual(
																										#     encoding.offset_mapping[1],
																										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																										# )

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(1, 1 + len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(1 + len(_a				) + 1, 1 + len(_a				) + 1 + len(_a				))      ,)

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, 1 + len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(1 + len(_a				), 1 + len(_a				) + 1 + len(_a				))      ,)

																										a__					=					self.rust_tokenizer_class.from_pretrained(
																										    _a      ,use_fast=_a      ,add_prefix_space=_a      ,trim_offsets=_a				)
																										a__					=					tokenizer_r(_a      ,return_offsets_mapping=_a      ,add_special_tokens=_a				)
																										self.assertEqual(encoding.offset_mapping[0]      ,(0, 1 + len(_a				))				)
																										self.assertEqual(
																										    encoding.offset_mapping[1]      ,(1 + len(_a				), 1 + len(_a				) + 1 + len(_a				))      ,)








import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


snake_case							:	Dict					      =				get_logger(__name__)


snake_case							:	str					      =				r'''
    Args:
        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):
            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam
            search or log softmax for each vocabulary token when using beam search
        kwargs (`Dict[str, Any]`, *optional*):
            Additional logits processor specific kwargs.

    Return:
        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.

'''







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :List[str]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       (lowerCamelCase_				):
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int      ,**__snake_case       :Any				) ->   jnp.ndarray:
												for processor in self:
																			a__					=					inspect.signature(processor.__call__				).parameters
																			if len(__snake_case				) > 3:
																										if not all(arg in kwargs for arg in list(function_args.keys()				)[2:]				):
																																	raise ValueError(
																																	    F'Make sure that all the required parameters: {list(function_args.keys()				)} for '
																																	    F'{processor.__class__} are passed to the logits processor.'				)
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case      ,**__snake_case				)
																			else:
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case				)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :float				) ->   Tuple:
												if not isinstance(__snake_case      ,__snake_case				) or not (temperature > 0):
																			raise ValueError(F'`temperature` has to be a strictly positive float, but is {temperature}'				)

												a__					=					temperature



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores / self.temperature
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :float      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Dict:
												if not isinstance(__snake_case      ,__snake_case				) or (top_p < 0 or top_p > 1.0):
																			raise ValueError(F'`top_p` has to be a float > 0 and < 1, but is {top_p}'				)
												if not isinstance(__snake_case      ,__snake_case				) or (min_tokens_to_keep < 1):
																			raise ValueError(F'`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}'				)

												a__					=					top_p
												a__					=					filter_value
												a__					=					min_tokens_to_keep



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					lax.top_k(__snake_case      ,scores.shape[-1]				)

												a__					=					jnp.full_like(__snake_case      ,self.filter_value				)
												a__					=					jax.nn.softmax(__snake_case      ,axis=-1				).cumsum(axis=-1				)
												a__					=					cumulative_probs < self.top_p

												# include the token that is higher than top_p as well
												a__					=					jnp.roll(__snake_case      ,1				)
												score_mask |= score_mask.at[:, 0].set(__snake_case				)

												# min tokens to keep
												a__					=					score_mask.at[:, : self.min_tokens_to_keep].set(__snake_case				)

												a__					=					jnp.where(__snake_case      ,__snake_case      ,__snake_case				)
												a__					=					jax.lax.sort_key_val(__snake_case      ,__snake_case				)[-1]

												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :int      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Any:
												if not isinstance(__snake_case      ,__snake_case				) or top_k <= 0:
																			raise ValueError(F'`top_k` has to be a strictly positive integer, but is {top_k}'				)

												a__					=					max(__snake_case      ,__snake_case				)
												a__					=					filter_value



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					scores.shape
												a__					=					jnp.full(batch_size * vocab_size      ,self.filter_value				)

												a__					=					min(self.top_k      ,scores.shape[-1]				)  # Safety check
												a__				, a__					=					lax.top_k(__snake_case      ,__snake_case				)
												a__					=					jnp.broadcast_to((jnp.arange(__snake_case				) * vocab_size)[:, None]      ,(batch_size, topk)				).flatten()
												a__					=					topk_scores.flatten()
												a__					=					topk_indices.flatten() + shift

												a__					=					next_scores_flat.at[topk_indices_flat].set(__snake_case				)
												a__					=					next_scores_flat.reshape(__snake_case      ,__snake_case				)
												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :int      ,__snake_case       :int				) ->   str:
												a__					=					bos_token_id



					def __call__(				self       :List[Any]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.bos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Union[str, Any]      ,__snake_case       :int      ,__snake_case       :int				) ->   List[Any]:
												a__					=					max_length
												a__					=					eos_token_id



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - self.max_length + 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.eos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :int      ,__snake_case       :int				) ->   List[str]:
												if not isinstance(__snake_case      ,__snake_case				) or min_length < 0:
																			raise ValueError(F'`min_length` has to be a positive integer, but is {min_length}'				)

												if not isinstance(__snake_case      ,__snake_case				) or eos_token_id < 0:
																			raise ValueError(F'`eos_token_id` has to be a positive integer, but is {eos_token_id}'				)

												a__					=					min_length
												a__					=					eos_token_id



					def __call__(				self       :Any      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												# create boolean flag to decide if min length penalty should be applied
												a__					=					1 - jnp.clip(cur_len - self.min_length      ,0      ,1				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.eos_token_id].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Optional[int]      ,__snake_case       :List[str]      ,__snake_case       :Optional[int]				) ->   Tuple:
												a__					=					list(__snake_case				)
												a__					=					begin_index



					def __call__(				self       :str      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :int				) ->   str:
												a__					=					1 - jnp.bool_(cur_len - self.begin_index				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.begin_suppress_tokens].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :list				) ->   List[Any]:
												a__					=					list(__snake_case				)



					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores.at[..., self.suppress_tokens].set(-float('inf'				)				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Dict      ,__snake_case       :Optional[int]				) ->   Union[str, Any]:
												a__					=					dict(__snake_case				)
												# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
												# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
												# Indexes without forced tokens will have a negative value.
												a__					=					jnp.ones((max(force_token_map.keys()				) + 1)      ,dtype=jnp.intaa				) * -1
												for index, token in force_token_map.items():
																			if token is not None:
																										a__					=					force_token_array.at[index].set(__snake_case				)
												a__					=					jnp.intaa(__snake_case				)



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												def _force_token(__snake_case       :Optional[Any]				):
																			a__					=					scores.shape[0]
																			a__					=					self.force_token_array[generation_idx]

																			a__					=					jnp.ones_like(__snake_case      ,dtype=scores.dtype				) * -float('inf'				)
																			a__					=					jnp.zeros((batch_size, 1)      ,dtype=scores.dtype				)
																			a__					=					lax.dynamic_update_slice(__snake_case      ,__snake_case      ,(0, current_token)				)
																			return new_scores

												a__					=					lax.cond(
												    cur_len >= self.force_token_array.shape[0]      ,lambda: scores      ,lambda: lax.cond(
												        self.force_token_array[cur_len] >= 0      ,lambda: _force_token(__snake_case				)      ,lambda: scores      ,)      ,)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :List[Any]				) ->   Optional[int]:
												a__					=					generate_config.eos_token_id
												a__					=					generate_config.no_timestamps_token_id
												a__					=					generate_config.no_timestamps_token_id + 1

												a__					=					decoder_input_length + 1

												if generate_config.is_multilingual:
																			# room for language token and task token
																			self.begin_index += 2
												if hasattr(__snake_case      ,'max_initial_timestamp_index'				):
																			a__					=					generate_config.max_initial_timestamp_index
												else:
																			a__					=					model_config.vocab_size
												if self.max_initial_timestamp_index is None:
																			a__					=					model_config.vocab_size



					def __call__(				self       :Any      ,__snake_case       :List[Any]      ,__snake_case       :Optional[int]      ,__snake_case       :Optional[Any]				) ->   Tuple:
												# suppress <|notimestamps|> which is handled by without_timestamps
												a__					=					scores.at[:, self.no_timestamps_token_id].set(-float('inf'				)				)

												def handle_pairs(__snake_case       :List[str]      ,__snake_case       :Union[str, Any]				):
																			a__					=					jnp.where((cur_len - self.begin_index) >= 1      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 1] >= self.timestamp_begin      ,True and last_was_timestamp      ,__snake_case      ,)

																			a__					=					jnp.where((cur_len - self.begin_index) < 2      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 2] >= self.timestamp_begin      ,__snake_case      ,__snake_case      ,)

																			return jnp.where(
																			    __snake_case      ,jnp.where(
																			        penultimate_was_timestamp > 0      ,scores_k.at[self.timestamp_begin :].set(-float('inf'				)				)      ,scores_k.at[: self.eos_token_id].set(-float('inf'				)				)      ,)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												a__					=					jnp.where(cur_len == self.begin_index      ,__snake_case      ,__snake_case				)
												a__					=					jnp.where(
												    self.max_initial_timestamp_index is not None      ,True and apply_max_initial_timestamp      ,__snake_case      ,)

												a__					=					self.timestamp_begin + self.max_initial_timestamp_index

												a__					=					jnp.where(
												    __snake_case      ,scores.at[:, last_allowed + 1 :].set(-float('inf'				)				)      ,__snake_case      ,)

												# if sum of probability over timestamps is above any other token, sample timestamp
												a__					=					jax.nn.log_softmax(__snake_case      ,axis=-1				)

												def handle_cumulative_probs(__snake_case       :Dict      ,__snake_case       :List[Any]				):
																			a__					=					jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]      ,axis=-1				)
																			a__					=					jnp.max(logprobs_k[: self.timestamp_begin]				)
																			return jnp.where(
																			    timestamp_logprob > max_text_token_logprob      ,scores_k.at[: self.timestamp_begin].set(-float('inf'				)				)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												return scores




