






"""simple docstring"""




import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
					if is_torch_available():
										import torch
					if is_tf_available():
										import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__snake_case      :      Any             =       logging.get_logger(__name__)

__snake_case      :      str             =       {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__snake_case      :      Optional[int]             =       {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

__snake_case      :      Union[str, Any]             =       {
    "Salesforce/codegen-350M-mono": 20_48,
}


class       __SCREAMING_SNAKE_CASE   (				__lowerCAmelCase):
						_SCREAMING_SNAKE_CASE		:    Dict						=   VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE		:    Optional[Any]						=   PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE		:    Optional[Any]						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE		:    Any						=   ["input_ids", "attention_mask"]
						_SCREAMING_SNAKE_CASE		:    Dict						=   CodeGenTokenizer






						def __init__(		self       ,  _UpperCamelCase=None       ,  _UpperCamelCase=None       ,  _UpperCamelCase=None       ,  _UpperCamelCase="<|endoftext|>"       ,  _UpperCamelCase="<|endoftext|>"       ,  _UpperCamelCase="<|endoftext|>"       ,  _UpperCamelCase=False       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												super().__init__(
												    lowerCamelCase__       ,  lowerCamelCase__       ,  tokenizer_file=lowerCamelCase__       ,  unk_token=lowerCamelCase__       ,  bos_token=lowerCamelCase__       ,  eos_token=lowerCamelCase__       ,  add_prefix_space=lowerCamelCase__       ,  **lowerCamelCase__       ,  )

												if kwargs.pop('add_bos_token'       ,  lowerCamelCase__				):
																		lowerCAmelCase__        =  kwargs.pop('name_or_path'       ,  ''				)
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    F"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
																		    F"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.'				)

												lowerCAmelCase__        =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
												if pre_tok_state.get('add_prefix_space'       ,  lowerCamelCase__				) != add_prefix_space:
																		lowerCAmelCase__        =  getattr(lowerCamelCase__       ,  pre_tok_state.pop('type'				)				)
																		lowerCAmelCase__        =  add_prefix_space
																		lowerCAmelCase__        =  pre_tok_class(**lowerCamelCase__				)

												lowerCAmelCase__        =  add_prefix_space






						def  UpperCamelCase__ (		self       ,  *_UpperCamelCase       ,  **_UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  kwargs.get('is_split_into_words'       ,  lowerCamelCase__				)
												assert self.add_prefix_space or not is_split_into_words, (
												    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*lowerCamelCase__       ,  **lowerCamelCase__				)






						def  UpperCamelCase__ (		self       ,  *_UpperCamelCase       ,  **_UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  kwargs.get('is_split_into_words'       ,  lowerCamelCase__				)

												assert self.add_prefix_space or not is_split_into_words, (
												    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*lowerCamelCase__       ,  **lowerCamelCase__				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = None				):






												"""simple docstring"""
												lowerCAmelCase__        =  self._tokenizer.model.save(lowerCamelCase__       ,  name=lowerCamelCase__				)
												return tuple(lowerCamelCase__				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = False       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												lowerCAmelCase__        =  super().decode(
												    token_ids=lowerCamelCase__       ,  skip_special_tokens=lowerCamelCase__       ,  clean_up_tokenization_spaces=lowerCamelCase__       ,  **lowerCamelCase__       ,  )

												if truncate_before_pattern is not None and len(lowerCamelCase__				) > 0:
																		lowerCAmelCase__        =  self.truncate(lowerCamelCase__       ,  lowerCamelCase__				)

												return decoded_text







						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase				):






												"""simple docstring"""






												def find_re(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase				):
																		lowerCAmelCase__        =  pattern.search(lowerCamelCase__       ,  lowerCamelCase__				)
																		return m.start() if m else -1

												lowerCAmelCase__        =  [re.compile(lowerCamelCase__       ,  re.MULTILINE				) for pattern in truncate_before_pattern]

												lowerCAmelCase__        =  list(re.finditer('^print'       ,  lowerCamelCase__       ,  re.MULTILINE				)				)

												if len(lowerCamelCase__				) > 1:
																		lowerCAmelCase__        =  completion[: prints[1].start()]

												lowerCAmelCase__        =  list(re.finditer('^def'       ,  lowerCamelCase__       ,  re.MULTILINE				)				)

												if len(lowerCamelCase__				) > 1:
																		lowerCAmelCase__        =  completion[: defs[1].start()]

												lowerCAmelCase__        =  0

												lowerCAmelCase__        =  [
												    pos for pos in [find_re(lowerCamelCase__       ,  lowerCamelCase__       ,  lowerCamelCase__				) for terminal in terminals] if pos != -1
												]

												if len(lowerCamelCase__				) > 0:
																		return completion[: min(lowerCamelCase__				)]
												else:
																		return completion













from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__snake_case      :      Dict             =       {"""configuration_focalnet""": ["""FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP""", """FocalNetConfig"""]}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case      :      str             =       [
					    """FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """FocalNetForImageClassification""",
					    """FocalNetForMaskedImageModeling""",
					    """FocalNetBackbone""",
					    """FocalNetModel""",
					    """FocalNetPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_focalnet import FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP, FocalNetConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_focalnet import (
										    FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST,
										    FocalNetBackbone,
										    FocalNetForImageClassification,
										    FocalNetForMaskedImageModeling,
										    FocalNetModel,
										    FocalNetPreTrainedModel,
										)

else:
					import sys

					__snake_case      :      List[Any]             =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





