


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case							:	Optional[Any]					      =				logging.get_logger(__name__)


snake_case							:	str					      =				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
snake_case							:	Dict					      =				{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

snake_case							:	Union[str, Any]					      =				{
    '''facebook/bart-base''': 10_24,
    '''facebook/bart-large''': 10_24,
    '''facebook/bart-large-mnli''': 10_24,
    '''facebook/bart-large-cnn''': 10_24,
    '''facebook/bart-large-xsum''': 10_24,
    '''yjernite/bart_eli5''': 10_24,
}






@lru_cache()
def 							__lowercase       (			):
							a__					=					(
							    list(range(ord('!'       ) ,   ord('~'       ) + 1       )       ) + list(range(ord('¡'       ) ,   ord('¬'       ) + 1       )       ) + list(range(ord('®'       ) ,   ord('ÿ'       ) + 1       )       )
							)
							a__					=					bs[:]
							a__					=					0
							for b in range(2**8       ):
														if b not in bs:
																					bs.append(__lowerCAmelCase       )
																					cs.append(2**8 + n       )
																					n += 1
							a__					=					[chr(__lowerCAmelCase       ) for n in cs]
							return dict(zip(__lowerCAmelCase ,   __lowerCAmelCase       )       )






def 							__lowercase       (			__lowerCAmelCase							:    List[Any]       ):
							a__					=					set()
							a__					=					word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)       )
														a__					=					char
							return pairs







class 	snake_case_       (lowerCamelCase_				):
					UpperCAmelCase__ :   Any   				=      VOCAB_FILES_NAMES
					UpperCAmelCase__ :   List[str]   				=      PRETRAINED_VOCAB_FILES_MAP
					UpperCAmelCase__ :   int   				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCAmelCase__ :   Union[str, Any]   				=      ['''input_ids''', '''attention_mask''']
					def __init__(				self       :int      ,__snake_case       :int      ,__snake_case       :int      ,__snake_case       :Any="replace"      ,__snake_case       :Optional[int]="<s>"      ,__snake_case       :int="</s>"      ,__snake_case       :Optional[Any]="</s>"      ,__snake_case       :Union[str, Any]="<s>"      ,__snake_case       :List[str]="<unk>"      ,__snake_case       :Union[str, Any]="<pad>"      ,__snake_case       :List[Any]="<mask>"      ,__snake_case       :Optional[Any]=False      ,**__snake_case       :str      ,) ->   List[Any]:
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else bos_token
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else eos_token
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else sep_token
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else cls_token
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else unk_token
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else pad_token

												# Mask token behave like a normal word, i.e. include the space before it
												a__					=					AddedToken(__snake_case      ,lstrip=__snake_case      ,rstrip=__snake_case				) if isinstance(__snake_case      ,__snake_case				) else mask_token

												super().__init__(
												    errors=__snake_case      ,bos_token=__snake_case      ,eos_token=__snake_case      ,unk_token=__snake_case      ,sep_token=__snake_case      ,cls_token=__snake_case      ,pad_token=__snake_case      ,mask_token=__snake_case      ,add_prefix_space=__snake_case      ,**__snake_case      ,)

												with open(__snake_case      ,encoding='utf-8'				) as vocab_handle:
																			a__					=					json.load(__snake_case				)
												a__					=					{v: k for k, v in self.encoder.items()}
												a__					=					errors  # how to handle errors in decoding
												a__					=					bytes_to_unicode()
												a__					=					{v: k for k, v in self.byte_encoder.items()}
												with open(__snake_case      ,encoding='utf-8'				) as merges_handle:
																			a__					=					merges_handle.read().split('\n'				)[1:-1]
												a__					=					[tuple(merge.split()				) for merge in bpe_merges]
												a__					=					dict(zip(__snake_case      ,range(len(__snake_case				)				)				)				)
												a__					=					{}
												a__					=					add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												a__					=					re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'				)
					@property
					def 			lowerCamelCase__(				self       :int				) ->   Optional[int]:
												return len(self.encoder				)
					def 			lowerCamelCase__(				self       :Optional[int]				) ->   Any:
												return dict(self.encoder      ,**self.added_tokens_encoder				)
					def 			lowerCamelCase__(				self       :Tuple      ,__snake_case       :List[Any]				) ->   List[Any]:
												if token in self.cache:
																			return self.cache[token]
												a__					=					tuple(__snake_case				)
												a__					=					get_pairs(__snake_case				)

												if not pairs:
																			return token

												while True:
																			a__					=					min(__snake_case      ,key=lambda __snake_case				: self.bpe_ranks.get(__snake_case      ,float('inf'				)				)				)
																			if bigram not in self.bpe_ranks:
																										break
																			a__				, a__					=					bigram
																			a__					=					[]
																			a__					=					0
																			while i < len(__snake_case				):
																										try:
																																	a__					=					word.index(__snake_case      ,__snake_case				)
																										except ValueError:
																																	new_word.extend(word[i:]				)
																																	break
																										else:
																																	new_word.extend(word[i:j]				)
																																	a__					=					j

																										if word[i] == first and i < len(__snake_case				) - 1 and word[i + 1] == second:
																																	new_word.append(first + second				)
																																	i += 2
																										else:
																																	new_word.append(word[i]				)
																																	i += 1
																			a__					=					tuple(__snake_case				)
																			a__					=					new_word
																			if len(__snake_case				) == 1:
																										break
																			else:
																										a__					=					get_pairs(__snake_case				)
												a__					=					' '.join(__snake_case				)
												a__					=					word
												return word
					def 			lowerCamelCase__(				self       :List[Any]      ,__snake_case       :Optional[Any]				) ->   str:
												a__					=					[]
												for token in re.findall(self.pat      ,__snake_case				):
																			a__					=					''.join(
																			    self.byte_encoder[b] for b in token.encode('utf-8'				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case				).split(' '				)				)
												return bpe_tokens
					def 			lowerCamelCase__(				self       :List[str]      ,__snake_case       :Dict				) ->   List[Any]:
												return self.encoder.get(__snake_case      ,self.encoder.get(self.unk_token				)				)
					def 			lowerCamelCase__(				self       :str      ,__snake_case       :Dict				) ->   int:
												return self.decoder.get(__snake_case				)
					def 			lowerCamelCase__(				self       :Any      ,__snake_case       :List[str]				) ->   Dict:
												a__					=					''.join(__snake_case				)
												a__					=					bytearray([self.byte_decoder[c] for c in text]				).decode('utf-8'      ,errors=self.errors				)
												return text
					def 			lowerCamelCase__(				self       :str      ,__snake_case       :str      ,__snake_case       :Optional[str] = None				) ->   Tuple[str]:
												if not os.path.isdir(__snake_case				):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
																			return
												a__					=					os.path.join(
												    __snake_case      ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
												a__					=					os.path.join(
												    __snake_case      ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']				)

												with open(__snake_case      ,'w'      ,encoding='utf-8'				) as f:
																			f.write(json.dumps(self.encoder      ,indent=2      ,sort_keys=__snake_case      ,ensure_ascii=__snake_case				) + '\n'				)

												a__					=					0
												with open(__snake_case      ,'w'      ,encoding='utf-8'				) as writer:
																			writer.write('#version: 0.2\n'				)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,key=lambda __snake_case				: kv[1]				):
																										if index != token_index:
																																	logger.warning(
																																	    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																																	    ' Please check that the tokenizer is not corrupted!'				)
																																	a__					=					token_index
																										writer.write(' '.join(__snake_case				) + '\n'				)
																										index += 1

												return vocab_file, merge_file
					def 			lowerCamelCase__(				self       :List[Any]      ,__snake_case       :List[int]      ,__snake_case       :Optional[List[int]] = None				) ->   List[int]:
												if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												a__					=					[self.cls_token_id]
												a__					=					[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep
					def 			lowerCamelCase__(				self       :Union[str, Any]      ,__snake_case       :List[int]      ,__snake_case       :Optional[List[int]] = None      ,__snake_case       :bool = False				) ->   List[int]:
												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__snake_case      ,token_ids_a=__snake_case      ,already_has_special_tokens=__snake_case				)

												if token_ids_a is None:
																			return [1] + ([0] * len(__snake_case				)) + [1]
												return [1] + ([0] * len(__snake_case				)) + [1, 1] + ([0] * len(__snake_case				)) + [1]
					def 			lowerCamelCase__(				self       :List[Any]      ,__snake_case       :List[int]      ,__snake_case       :Optional[List[int]] = None				) ->   List[int]:
												a__					=					[self.sep_token_id]
												a__					=					[self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep				) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]



					def 			lowerCamelCase__(				self       :Optional[int]      ,__snake_case       :int      ,__snake_case       :Optional[Any]=False      ,**__snake_case       :Tuple				) ->   Dict:
												a__					=					kwargs.pop('add_prefix_space'      ,self.add_prefix_space				)
												if (is_split_into_words or add_prefix_space) and (len(__snake_case				) > 0 and not text[0].isspace()):
																			a__					=					' ' + text
												return (text, kwargs)








import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


snake_case							:	Dict					      =				get_logger(__name__)


snake_case							:	str					      =				r'''
    Args:
        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):
            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam
            search or log softmax for each vocabulary token when using beam search
        kwargs (`Dict[str, Any]`, *optional*):
            Additional logits processor specific kwargs.

    Return:
        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.

'''







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :List[str]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       (lowerCamelCase_				):
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int      ,**__snake_case       :Any				) ->   jnp.ndarray:
												for processor in self:
																			a__					=					inspect.signature(processor.__call__				).parameters
																			if len(__snake_case				) > 3:
																										if not all(arg in kwargs for arg in list(function_args.keys()				)[2:]				):
																																	raise ValueError(
																																	    F'Make sure that all the required parameters: {list(function_args.keys()				)} for '
																																	    F'{processor.__class__} are passed to the logits processor.'				)
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case      ,**__snake_case				)
																			else:
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case				)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :float				) ->   Tuple:
												if not isinstance(__snake_case      ,__snake_case				) or not (temperature > 0):
																			raise ValueError(F'`temperature` has to be a strictly positive float, but is {temperature}'				)

												a__					=					temperature



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores / self.temperature
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :float      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Dict:
												if not isinstance(__snake_case      ,__snake_case				) or (top_p < 0 or top_p > 1.0):
																			raise ValueError(F'`top_p` has to be a float > 0 and < 1, but is {top_p}'				)
												if not isinstance(__snake_case      ,__snake_case				) or (min_tokens_to_keep < 1):
																			raise ValueError(F'`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}'				)

												a__					=					top_p
												a__					=					filter_value
												a__					=					min_tokens_to_keep



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					lax.top_k(__snake_case      ,scores.shape[-1]				)

												a__					=					jnp.full_like(__snake_case      ,self.filter_value				)
												a__					=					jax.nn.softmax(__snake_case      ,axis=-1				).cumsum(axis=-1				)
												a__					=					cumulative_probs < self.top_p

												# include the token that is higher than top_p as well
												a__					=					jnp.roll(__snake_case      ,1				)
												score_mask |= score_mask.at[:, 0].set(__snake_case				)

												# min tokens to keep
												a__					=					score_mask.at[:, : self.min_tokens_to_keep].set(__snake_case				)

												a__					=					jnp.where(__snake_case      ,__snake_case      ,__snake_case				)
												a__					=					jax.lax.sort_key_val(__snake_case      ,__snake_case				)[-1]

												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :int      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Any:
												if not isinstance(__snake_case      ,__snake_case				) or top_k <= 0:
																			raise ValueError(F'`top_k` has to be a strictly positive integer, but is {top_k}'				)

												a__					=					max(__snake_case      ,__snake_case				)
												a__					=					filter_value



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					scores.shape
												a__					=					jnp.full(batch_size * vocab_size      ,self.filter_value				)

												a__					=					min(self.top_k      ,scores.shape[-1]				)  # Safety check
												a__				, a__					=					lax.top_k(__snake_case      ,__snake_case				)
												a__					=					jnp.broadcast_to((jnp.arange(__snake_case				) * vocab_size)[:, None]      ,(batch_size, topk)				).flatten()
												a__					=					topk_scores.flatten()
												a__					=					topk_indices.flatten() + shift

												a__					=					next_scores_flat.at[topk_indices_flat].set(__snake_case				)
												a__					=					next_scores_flat.reshape(__snake_case      ,__snake_case				)
												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :int      ,__snake_case       :int				) ->   str:
												a__					=					bos_token_id



					def __call__(				self       :List[Any]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.bos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Union[str, Any]      ,__snake_case       :int      ,__snake_case       :int				) ->   List[Any]:
												a__					=					max_length
												a__					=					eos_token_id



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - self.max_length + 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.eos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :int      ,__snake_case       :int				) ->   List[str]:
												if not isinstance(__snake_case      ,__snake_case				) or min_length < 0:
																			raise ValueError(F'`min_length` has to be a positive integer, but is {min_length}'				)

												if not isinstance(__snake_case      ,__snake_case				) or eos_token_id < 0:
																			raise ValueError(F'`eos_token_id` has to be a positive integer, but is {eos_token_id}'				)

												a__					=					min_length
												a__					=					eos_token_id



					def __call__(				self       :Any      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												# create boolean flag to decide if min length penalty should be applied
												a__					=					1 - jnp.clip(cur_len - self.min_length      ,0      ,1				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.eos_token_id].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Optional[int]      ,__snake_case       :List[str]      ,__snake_case       :Optional[int]				) ->   Tuple:
												a__					=					list(__snake_case				)
												a__					=					begin_index



					def __call__(				self       :str      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :int				) ->   str:
												a__					=					1 - jnp.bool_(cur_len - self.begin_index				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.begin_suppress_tokens].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :list				) ->   List[Any]:
												a__					=					list(__snake_case				)



					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores.at[..., self.suppress_tokens].set(-float('inf'				)				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Dict      ,__snake_case       :Optional[int]				) ->   Union[str, Any]:
												a__					=					dict(__snake_case				)
												# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
												# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
												# Indexes without forced tokens will have a negative value.
												a__					=					jnp.ones((max(force_token_map.keys()				) + 1)      ,dtype=jnp.intaa				) * -1
												for index, token in force_token_map.items():
																			if token is not None:
																										a__					=					force_token_array.at[index].set(__snake_case				)
												a__					=					jnp.intaa(__snake_case				)



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												def _force_token(__snake_case       :Optional[Any]				):
																			a__					=					scores.shape[0]
																			a__					=					self.force_token_array[generation_idx]

																			a__					=					jnp.ones_like(__snake_case      ,dtype=scores.dtype				) * -float('inf'				)
																			a__					=					jnp.zeros((batch_size, 1)      ,dtype=scores.dtype				)
																			a__					=					lax.dynamic_update_slice(__snake_case      ,__snake_case      ,(0, current_token)				)
																			return new_scores

												a__					=					lax.cond(
												    cur_len >= self.force_token_array.shape[0]      ,lambda: scores      ,lambda: lax.cond(
												        self.force_token_array[cur_len] >= 0      ,lambda: _force_token(__snake_case				)      ,lambda: scores      ,)      ,)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :List[Any]				) ->   Optional[int]:
												a__					=					generate_config.eos_token_id
												a__					=					generate_config.no_timestamps_token_id
												a__					=					generate_config.no_timestamps_token_id + 1

												a__					=					decoder_input_length + 1

												if generate_config.is_multilingual:
																			# room for language token and task token
																			self.begin_index += 2
												if hasattr(__snake_case      ,'max_initial_timestamp_index'				):
																			a__					=					generate_config.max_initial_timestamp_index
												else:
																			a__					=					model_config.vocab_size
												if self.max_initial_timestamp_index is None:
																			a__					=					model_config.vocab_size



					def __call__(				self       :Any      ,__snake_case       :List[Any]      ,__snake_case       :Optional[int]      ,__snake_case       :Optional[Any]				) ->   Tuple:
												# suppress <|notimestamps|> which is handled by without_timestamps
												a__					=					scores.at[:, self.no_timestamps_token_id].set(-float('inf'				)				)

												def handle_pairs(__snake_case       :List[str]      ,__snake_case       :Union[str, Any]				):
																			a__					=					jnp.where((cur_len - self.begin_index) >= 1      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 1] >= self.timestamp_begin      ,True and last_was_timestamp      ,__snake_case      ,)

																			a__					=					jnp.where((cur_len - self.begin_index) < 2      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 2] >= self.timestamp_begin      ,__snake_case      ,__snake_case      ,)

																			return jnp.where(
																			    __snake_case      ,jnp.where(
																			        penultimate_was_timestamp > 0      ,scores_k.at[self.timestamp_begin :].set(-float('inf'				)				)      ,scores_k.at[: self.eos_token_id].set(-float('inf'				)				)      ,)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												a__					=					jnp.where(cur_len == self.begin_index      ,__snake_case      ,__snake_case				)
												a__					=					jnp.where(
												    self.max_initial_timestamp_index is not None      ,True and apply_max_initial_timestamp      ,__snake_case      ,)

												a__					=					self.timestamp_begin + self.max_initial_timestamp_index

												a__					=					jnp.where(
												    __snake_case      ,scores.at[:, last_allowed + 1 :].set(-float('inf'				)				)      ,__snake_case      ,)

												# if sum of probability over timestamps is above any other token, sample timestamp
												a__					=					jax.nn.log_softmax(__snake_case      ,axis=-1				)

												def handle_cumulative_probs(__snake_case       :Dict      ,__snake_case       :List[Any]				):
																			a__					=					jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]      ,axis=-1				)
																			a__					=					jnp.max(logprobs_k[: self.timestamp_begin]				)
																			return jnp.where(
																			    timestamp_logprob > max_text_token_logprob      ,scores_k.at[: self.timestamp_begin].set(-float('inf'				)				)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												return scores




