


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
			if is_torch_available():
						import torch
			if is_tf_available():
						import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_snake_case					:		Union[str, Any]      =    logging.get_logger(__name__)

_snake_case					:		Union[str, Any]      =    {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_snake_case					:		Dict      =    {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

_snake_case					:		Union[str, Any]      =    {
    "Salesforce/codegen-350M-mono": 2_048,
}

class     a		(_lowerCAmelCase				):




							"""simple docstring"""


							__UpperCAmelCase   :							Any					  =  VOCAB_FILES_NAMES
							__UpperCAmelCase   :							str					  =  PRETRAINED_VOCAB_FILES_MAP
							__UpperCAmelCase   :							Optional[Any]					  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__UpperCAmelCase   :							int					  =  ["input_ids", "attention_mask"]
							__UpperCAmelCase   :							Tuple					  =  CodeGenTokenizer

							def __init__(							self	:  List[str]   ,     lowerCamelCase	:  Union[str, Any]=None   ,     lowerCamelCase	:  List[str]=None   ,     lowerCamelCase	:  Union[str, Any]=None   ,     lowerCamelCase	:  Optional[Any]="<|endoftext|>"   ,     lowerCamelCase	:  Dict="<|endoftext|>"   ,     lowerCamelCase	:  Any="<|endoftext|>"   ,     lowerCamelCase	:  Optional[Any]=False   ,     **lowerCamelCase	:  int   ,     )     ->   Dict:
														super().__init__(
														    lowerCamelCase   ,     lowerCamelCase   ,     tokenizer_file=lowerCamelCase   ,     unk_token=lowerCamelCase   ,     bos_token=lowerCamelCase   ,     eos_token=lowerCamelCase   ,     add_prefix_space=lowerCamelCase   ,     **lowerCamelCase   ,     )

														if kwargs.pop("add_bos_token"   ,     lowerCamelCase    ):
																					__snake_case		:      List[str]										=       kwargs.pop("name_or_path"   ,     ""    )
																					raise ValueError(
																					    "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
																					    "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
																					    F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
																					    F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
																					    "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
																					    " so that the fast tokenizer works correctly."    )

														__snake_case		:      Dict										=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
														if pre_tok_state.get("add_prefix_space"   ,     lowerCamelCase    ) != add_prefix_space:
																					__snake_case		:      List[str]										=       getattr(lowerCamelCase   ,     pre_tok_state.pop("type"    )    )
																					__snake_case		:      Tuple										=       add_prefix_space
																					__snake_case		:      int										=       pre_tok_class(**lowerCamelCase    )

														__snake_case		:      Optional[int]										=       add_prefix_space

							def    __snake_case (							self	:  int   ,     *lowerCamelCase	:  List[Any]   ,     **lowerCamelCase	:  List[Any]    )     ->   BatchEncoding:
														__snake_case		:      Optional[Any]										=       kwargs.get("is_split_into_words"   ,     lowerCamelCase    )
														assert self.add_prefix_space or not is_split_into_words, (
														    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*lowerCamelCase   ,     **lowerCamelCase    )

							def    __snake_case (							self	:  Optional[int]   ,     *lowerCamelCase	:  List[Any]   ,     **lowerCamelCase	:  Union[str, Any]    )     ->   BatchEncoding:
														__snake_case		:      List[str]										=       kwargs.get("is_split_into_words"   ,     lowerCamelCase    )

														assert self.add_prefix_space or not is_split_into_words, (
														    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*lowerCamelCase   ,     **lowerCamelCase    )

							def    __snake_case (							self	:  List[str]   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  Optional[str] = None    )     ->   Tuple[str]:
														__snake_case		:      List[Any]										=       self._tokenizer.model.save(lowerCamelCase   ,     name=lowerCamelCase    )
														return tuple(lowerCamelCase    )

							def    __snake_case (							self	:  Dict   ,     lowerCamelCase	:  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,     lowerCamelCase	:  bool = False   ,     lowerCamelCase	:  bool = None   ,     lowerCamelCase	:  Optional[List[str]] = None   ,     **lowerCamelCase	:  Tuple   ,     )     ->   str:
														__snake_case		:      Any										=       super().decode(
														    token_ids=lowerCamelCase   ,     skip_special_tokens=lowerCamelCase   ,     clean_up_tokenization_spaces=lowerCamelCase   ,     **lowerCamelCase   ,     )

														if truncate_before_pattern is not None and len(lowerCamelCase    ) > 0:
																					__snake_case		:      Union[str, Any]										=       self.truncate(lowerCamelCase   ,     lowerCamelCase    )

														return decoded_text






							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  Tuple   ,     lowerCamelCase	:  List[str]    )     ->   Optional[Any]:
														def find_re(lowerCamelCase	:  Union[str, Any]   ,     lowerCamelCase	:  List[Any]   ,     lowerCamelCase	:  Dict    ):
																					__snake_case		:      Optional[int]										=       pattern.search(lowerCamelCase   ,     lowerCamelCase    )
																					return m.start() if m else -1

														__snake_case		:      Dict										=       [re.compile(lowerCamelCase   ,     re.MULTILINE    ) for pattern in truncate_before_pattern]

														__snake_case		:      Optional[Any]										=       list(re.finditer("^print"   ,     lowerCamelCase   ,     re.MULTILINE    )    )

														if len(lowerCamelCase    ) > 1:
																					__snake_case		:      Optional[int]										=       completion[: prints[1].start()]

														__snake_case		:      List[str]										=       list(re.finditer("^def"   ,     lowerCamelCase   ,     re.MULTILINE    )    )

														if len(lowerCamelCase    ) > 1:
																					__snake_case		:      Optional[Any]										=       completion[: defs[1].start()]

														__snake_case		:      Dict										=       0

														__snake_case		:      Optional[Any]										=       [
														    pos for pos in [find_re(lowerCamelCase   ,     lowerCamelCase   ,     lowerCamelCase    ) for terminal in terminals] if pos != -1
														]

														if len(lowerCamelCase    ) > 0:
																					return completion[: min(lowerCamelCase    )]
														else:
																					return completion





from scipy.stats import spearmanr

import datasets


_snake_case					:		Optional[Any]      =    "\nThe Spearman rank-order correlation coefficient is a measure of the\nrelationship between two datasets. Like other correlation coefficients,\nthis one varies between -1 and +1 with 0 implying no correlation.\nPositive correlations imply that as data in dataset x increases, so\ndoes data in dataset y. Negative correlations imply that as x increases,\ny decreases. Correlations of -1 or +1 imply an exact monotonic relationship.\n\nUnlike the Pearson correlation, the Spearman correlation does not\nassume that both datasets are normally distributed.\n\nThe p-value roughly indicates the probability of an uncorrelated system\nproducing datasets that have a Spearman correlation at least as extreme\nas the one computed from these datasets. The p-values are not entirely\nreliable but are probably reasonable for datasets larger than 500 or so.\n"

_snake_case					:		Dict      =    "\nArgs:\n    predictions (`List[float]`): Predicted labels, as returned by a model.\n    references (`List[float]`): Ground truth labels.\n    return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns\n            only the spearmanr score. Defaults to `False`.\nReturns:\n    spearmanr (`float`): Spearman correlation coefficient.\n    p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input.\nExamples:\n    Example 1:\n        >>> spearmanr_metric = datasets.load_metric(\"spearmanr\")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4])\n        >>> print(results)\n        {'spearmanr': -0.7}\n\n    Example 2:\n        >>> spearmanr_metric = datasets.load_metric(\"spearmanr\")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5],\n        ...                                     predictions=[10, 9, 2.5, 6, 4],\n        ...                                     return_pvalue=True)\n        >>> print(results['spearmanr'])\n        -0.7\n        >>> print(round(results['spearmanr_pvalue'], 2))\n        0.19\n"

_snake_case					:		Optional[Any]      =    R"\\n@book{kokoska2000crc,\n  title={CRC standard probability and statistics tables and formulae},\n  author={Kokoska, Stephen and Zwillinger, Daniel},\n  year={2000},\n  publisher={Crc Press}\n}\n@article{2020SciPy-NMeth,\n  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and\n            Haberland, Matt and Reddy, Tyler and Cournapeau, David and\n            Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and\n            Bright, Jonathan and {van der Walt}, St{\'e}fan J. and\n            Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and\n            Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and\n            Kern, Robert and Larson, Eric and Carey, C J and\n            Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and\n            {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and\n            Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and\n            Harris, Charles R. and Archibald, Anne M. and\n            Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and\n            {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},\n  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific\n            Computing in Python}},\n  journal = {Nature Methods},\n  year    = {2020},\n  volume  = {17},\n  pages   = {261--272},\n  adsurl  = {https://rdcu.be/b08Wh},\n  doi     = {10.1038/s41592-019-0686-2},\n}\n"

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,    _KWARGS_DESCRIPTION				)
class     a		(datasets.Metric				):




							"""simple docstring"""


							def    __snake_case (							self	:  Optional[int]    )     ->   Any:
														return datasets.MetricInfo(
														    description=_DESCRIPTION   ,     citation=_CITATION   ,     inputs_description=_KWARGS_DESCRIPTION   ,     features=datasets.Features(
														        {
														            "predictions": datasets.Value("float"    ),
														            "references": datasets.Value("float"    ),
														        }    )   ,     reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"]   ,     )






							def    __snake_case (							self	:  Optional[Any]   ,     lowerCamelCase	:  Optional[int]   ,     lowerCamelCase	:  int   ,     lowerCamelCase	:  Tuple=False    )     ->   Dict:
														__snake_case		:      Optional[int]										=       spearmanr(lowerCamelCase   ,     lowerCamelCase    )
														if return_pvalue:
																					return {"spearmanr": results[0], "spearmanr_pvalue": results[1]}
														else:
																					return {"spearmanr": results[0]}

