"""simple docstring"""







import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_a					:					str=			logging.get_logger(__name__)

_a					:					Union[str, Any]=			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_a					:					int=			{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

_a					:					Union[str, Any]=			{
    "Salesforce/codegen-350M-mono": 2_048,
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Tuple   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				List[Any]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				Optional[Any]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				Dict   =		["""input_ids""", """attention_mask"""]
				UpperCAmelCase    :				Union[str, Any]   =		CodeGenTokenizer



				def __init__(self    :				str     ,  _A    :				str=None     ,  _A    :				Optional[Any]=None     ,  _A    :				Optional[Any]=None     ,  _A    :				Optional[int]="<|endoftext|>"     ,  _A    :				Optional[Any]="<|endoftext|>"     ,  _A    :				Optional[Any]="<|endoftext|>"     ,  _A    :				str=False     ,  **_A    :				Tuple     ,  )   ->     Union[str, Any]:
					super().__init__(
					    _A     ,  _A     ,  tokenizer_file=_A     ,  unk_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  add_prefix_space=_A     ,  **_A     ,  )

					if kwargs.pop('add_bos_token'     ,  _A):
						__snake_case      :							int						=			kwargs.pop('name_or_path'     ,  '')
						raise ValueError(
						    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
						    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
						    f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
						    f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
						    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
						    ' so that the fast tokenizer works correctly.')

					__snake_case      :							int						=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
					if pre_tok_state.get('add_prefix_space'     ,  _A) != add_prefix_space:
						__snake_case      :							List[str]						=			getattr(_A     ,  pre_tok_state.pop('type'))
						__snake_case      :							Tuple						=			add_prefix_space
						__snake_case      :							List[Any]						=			pre_tok_class(**_A)

					__snake_case      :							Any						=			add_prefix_space



				def  _lowercase					(self    :				Dict     ,  *_A    :				Any     ,  **_A    :				Optional[Any])   ->     BatchEncoding:
					__snake_case      :							Any						=			kwargs.get('is_split_into_words'     ,  _A)
					assert self.add_prefix_space or not is_split_into_words, (
					    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
					    "to use it with pretokenized inputs."
					)

					return super()._batch_encode_plus(*_A     ,  **_A)



				def  _lowercase					(self    :				List[str]     ,  *_A    :				Any     ,  **_A    :				Optional[int])   ->     BatchEncoding:
					__snake_case      :							Optional[Any]						=			kwargs.get('is_split_into_words'     ,  _A)

					assert self.add_prefix_space or not is_split_into_words, (
					    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
					    "to use it with pretokenized inputs."
					)

					return super()._encode_plus(*_A     ,  **_A)



				def  _lowercase					(self    :				int     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							str						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)



				def  _lowercase					(self    :				List[str]     ,  _A    :				Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]     ,  _A    :				bool = False     ,  _A    :				bool = None     ,  _A    :				Optional[List[str]] = None     ,  **_A    :				List[Any]     ,  )   ->     str:
					__snake_case      :							Tuple						=			super().decode(
					    token_ids=_A     ,  skip_special_tokens=_A     ,  clean_up_tokenization_spaces=_A     ,  **_A     ,  )

					if truncate_before_pattern is not None and len(_A) > 0:
						__snake_case      :							Union[str, Any]						=			self.truncate(_A     ,  _A)

					return decoded_text






				def  _lowercase					(self    :				int     ,  _A    :				Optional[Any]     ,  _A    :				int)   ->     Optional[Any]:
					def find_re(_A    :				str     ,  _A    :				List[str]     ,  _A    :				str):
						__snake_case      :							Any						=			pattern.search(_A     ,  _A)
						return m.start() if m else -1

					__snake_case      :							Tuple						=			[re.compile(_A     ,  re.MULTILINE) for pattern in truncate_before_pattern]

					__snake_case      :							Any						=			list(re.finditer('^print'     ,  _A     ,  re.MULTILINE))

					if len(_A) > 1:
						__snake_case      :							List[str]						=			completion[: prints[1].start()]

					__snake_case      :							Dict						=			list(re.finditer('^def'     ,  _A     ,  re.MULTILINE))

					if len(_A) > 1:
						__snake_case      :							List[Any]						=			completion[: defs[1].start()]

					__snake_case      :							List[str]						=			0

					__snake_case      :							str						=			[
					    pos for pos in [find_re(_A     ,  _A     ,  _A) for terminal in terminals] if pos != -1
					]

					if len(_A) > 0:
						return completion[: min(_A)]
					else:
						return completion






"""simple docstring"""







def   __UpperCAmelCase    (   UpperCAmelCase_   :    int      ,							UpperCAmelCase_   :    int  )							->      int:

	'''simple docstring'''





	while a != 0:
		__snake_case		,    __snake_case      :							Union[str, Any]						=			b % a, a
	return b

def   __UpperCAmelCase    (   UpperCAmelCase_   :    int      ,							UpperCAmelCase_   :    int  )							->      int:

	'''simple docstring'''





	if gcd(UpperCAmelCase_      ,							UpperCAmelCase_  ) != 1:
		__snake_case      :							Union[str, Any]						=			F"mod inverse of {a!r} and {m!r} does not exist"
		raise ValueError(UpperCAmelCase_  )
	__snake_case		,    __snake_case		,    __snake_case      :							List[str]						=			1, 0, a
	__snake_case		,    __snake_case		,    __snake_case      :							Dict						=			0, 1, m
	while va != 0:
		__snake_case      :							List[str]						=			ua // va
		__snake_case		,    __snake_case		,    __snake_case		,    __snake_case		,    __snake_case		,    __snake_case      :							List[str]						=			(ua - q * va), (ua - q * va), (ua - q * va), va, va, va
	return ua % m





