
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     lowercase		( __snake_case	,  unittest.TestCase		):
							lowercase_	:	Dict	 =CodeGenTokenizer
							lowercase_	:	Any	 =CodeGenTokenizerFast
							lowercase_	:	Union[str, Any]	 =True
							lowercase_	:	List[Any]	 ={"""add_prefix_space""": True}
							lowercase_	:	List[str]	 =False




							def 			A__				(    self):
												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												lowercase						     =						[
												    "l",
												    "o",
												    "w",
												    "e",
												    "r",
												    "s",
												    "t",
												    "i",
												    "d",
												    "n",
												    "\u0120",
												    "\u0120l",
												    "\u0120n",
												    "\u0120lo",
												    "\u0120low",
												    "er",
												    "\u0120lowest",
												    "\u0120newer",
												    "\u0120wider",
												    "<unk>",
												    "<|endoftext|>",
												]
												lowercase						     =						dict(zip(UpperCamelCase__       ,range(len(UpperCamelCase__))))
												lowercase						     =						["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
												lowercase						     =						{"unk_token": "<unk>"}

												lowercase						     =						os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''vocab_file'''])
												lowercase						     =						os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''merges_file'''])
												with open(self.vocab_file       ,'''w'''       ,encoding='''utf-8''') as fp:
																	fp.write(json.dumps(UpperCamelCase__) + '''\n''')
												with open(self.merges_file       ,'''w'''       ,encoding='''utf-8''') as fp:
																	fp.write('''\n'''.join(UpperCamelCase__))




							def 			A__				(    self       ,**A__):
												kwargs.update(self.special_tokens_map)
												return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,**UpperCamelCase__)




							def 			A__				(    self       ,**A__):
												kwargs.update(self.special_tokens_map)
												return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,**UpperCamelCase__)




							def 			A__				(    self       ,A__):
												lowercase						     =						"lower newer"
												lowercase						     =						"lower newer"
												return input_text, output_text




							def 			A__				(    self):
												lowercase						     =						CodeGenTokenizer(self.vocab_file       ,self.merges_file       ,**self.special_tokens_map)
												lowercase						     =						"lower newer"
												lowercase						     =						["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
												lowercase						     =						tokenizer.tokenize(UpperCamelCase__       ,add_prefix_space=UpperCamelCase__)
												self.assertListEqual(UpperCamelCase__       ,UpperCamelCase__)

												lowercase						     =						tokens + [tokenizer.unk_token]
												lowercase						     =						[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)       ,UpperCamelCase__)




							def 			A__				(    self):


												if not self.test_rust_tokenizer:
																	return

												lowercase						     =						self.get_tokenizer()
												lowercase						     =						self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__)

												lowercase						     =						"lower newer"

												# Testing tokenization
												lowercase						     =						tokenizer.tokenize(UpperCamelCase__       ,add_prefix_space=UpperCamelCase__)
												lowercase						     =						rust_tokenizer.tokenize(UpperCamelCase__)
												self.assertListEqual(UpperCamelCase__       ,UpperCamelCase__)

												# Testing conversion to ids without special tokens
												lowercase						     =						tokenizer.encode(UpperCamelCase__       ,add_special_tokens=UpperCamelCase__       ,add_prefix_space=UpperCamelCase__)
												lowercase						     =						rust_tokenizer.encode(UpperCamelCase__       ,add_special_tokens=UpperCamelCase__)
												self.assertListEqual(UpperCamelCase__       ,UpperCamelCase__)

												# Testing conversion to ids with special tokens
												lowercase						     =						self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__)
												lowercase						     =						tokenizer.encode(UpperCamelCase__       ,add_prefix_space=UpperCamelCase__)
												lowercase						     =						rust_tokenizer.encode(UpperCamelCase__)
												self.assertListEqual(UpperCamelCase__       ,UpperCamelCase__)

												# Testing the unknown token
												lowercase						     =						tokens + [rust_tokenizer.unk_token]
												lowercase						     =						[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
												self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase__)       ,UpperCamelCase__)




							def 			A__				(    self       ,*A__       ,**A__):
												pass




							def 			A__				(    self       ,A__=1_5):


												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
																						lowercase						     =						self.rust_tokenizer_class.from_pretrained(UpperCamelCase__       ,**UpperCamelCase__)

																						# Simple input
																						lowercase						     =						"This is a simple input"
																						lowercase						     =						["This is a simple input 1", "This is a simple input 2"]
																						lowercase						     =						("This is a simple input", "This is a pair")
																						lowercase						     =						[
																						    ("This is a simple input 1", "This is a simple input 2"),
																						    ("This is a simple pair 1", "This is a simple pair 2"),
																						]

																						# Simple input tests
																						self.assertRaises(UpperCamelCase__       ,tokenizer_r.encode       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length''')

																						# Simple input
																						self.assertRaises(UpperCamelCase__       ,tokenizer_r.encode_plus       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length''')

																						# Simple input
																						self.assertRaises(
																						    UpperCamelCase__       ,tokenizer_r.batch_encode_plus       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length'''       ,)

																						# Pair input
																						self.assertRaises(UpperCamelCase__       ,tokenizer_r.encode       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length''')

																						# Pair input
																						self.assertRaises(UpperCamelCase__       ,tokenizer_r.encode_plus       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length''')

																						# Pair input
																						self.assertRaises(
																						    UpperCamelCase__       ,tokenizer_r.batch_encode_plus       ,UpperCamelCase__       ,max_length=UpperCamelCase__       ,padding='''max_length'''       ,)




							def 			A__				(    self):
												lowercase						     =						CodeGenTokenizer.from_pretrained(self.tmpdirname       ,pad_token='''<pad>''')

												# Simple input
												lowercase						     =						"This is a simple input"
												lowercase						     =						["This is a simple input looooooooong", "This is a simple input"]
												lowercase						     =						("This is a simple input", "This is a pair")
												lowercase						     =						[
												    ("This is a simple input loooooong", "This is a simple input"),
												    ("This is a simple pair loooooong", "This is a simple pair"),
												]

												lowercase						     =						tokenizer.pad_token_id

												lowercase						     =						tokenizer(UpperCamelCase__       ,padding='''max_length'''       ,max_length=3_0       ,return_tensors='''np''')
												lowercase						     =						tokenizer(UpperCamelCase__       ,padding=UpperCamelCase__       ,truncate=UpperCamelCase__       ,return_tensors='''np''')
												lowercase						     =						tokenizer(*UpperCamelCase__       ,padding='''max_length'''       ,max_length=6_0       ,return_tensors='''np''')
												lowercase						     =						tokenizer(UpperCamelCase__       ,padding=UpperCamelCase__       ,truncate=UpperCamelCase__       ,return_tensors='''np''')

												# s
												# test single string max_length padding
												self.assertEqual(out_s['''input_ids'''].shape[-1]       ,3_0)
												self.assertTrue(pad_token_id in out_s['''input_ids'''])
												self.assertTrue(0 in out_s['''attention_mask'''])

												# s2
												# test automatic padding
												self.assertEqual(out_sa['''input_ids'''].shape[-1]       ,3_3)
												# long slice doesn't have padding
												self.assertFalse(pad_token_id in out_sa['''input_ids'''][0])
												self.assertFalse(0 in out_sa['''attention_mask'''][0])
												# short slice does have padding
												self.assertTrue(pad_token_id in out_sa['''input_ids'''][1])
												self.assertTrue(0 in out_sa['''attention_mask'''][1])

												# p
												# test single pair max_length padding
												self.assertEqual(out_p['''input_ids'''].shape[-1]       ,6_0)
												self.assertTrue(pad_token_id in out_p['''input_ids'''])
												self.assertTrue(0 in out_p['''attention_mask'''])

												# p2
												# test automatic padding pair
												self.assertEqual(out_pa['''input_ids'''].shape[-1]       ,5_2)
												# long slice pair doesn't have padding
												self.assertFalse(pad_token_id in out_pa['''input_ids'''][0])
												self.assertFalse(0 in out_pa['''attention_mask'''][0])
												# short slice pair does have padding
												self.assertTrue(pad_token_id in out_pa['''input_ids'''][1])
												self.assertTrue(0 in out_pa['''attention_mask'''][1])




							def 			A__				(    self):
												lowercase						     =						"$$$"
												lowercase						     =						CodeGenTokenizer.from_pretrained(self.tmpdirname       ,bos_token=UpperCamelCase__       ,add_bos_token=UpperCamelCase__)

												lowercase						     =						"This is a simple input"
												lowercase						     =						["This is a simple input 1", "This is a simple input 2"]

												lowercase						     =						tokenizer.bos_token_id

												lowercase						     =						tokenizer(UpperCamelCase__)
												lowercase						     =						tokenizer(UpperCamelCase__)

												self.assertEqual(out_s.input_ids[0]       ,UpperCamelCase__)
												self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

												lowercase						     =						tokenizer.decode(out_s.input_ids)
												lowercase						     =						tokenizer.batch_decode(out_sa.input_ids)

												self.assertEqual(decode_s.split()[0]       ,UpperCamelCase__)
												self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))




							@slow
							def 			A__				(    self):
												lowercase						     =						CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono''')

												lowercase						     =						"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
												lowercase						     =						"\nif len_a > len_b:      result = a\nelse:      result = b"

												lowercase						     =						tokenizer.encode(UpperCamelCase__)
												lowercase						     =						["^#", re.escape('''<|endoftext|>'''), "^'''", "^\"\"\"", "\n\n\n"]
												lowercase						     =						tokenizer.decode(UpperCamelCase__       ,truncate_before_pattern=UpperCamelCase__)
												self.assertEqual(UpperCamelCase__       ,UpperCamelCase__)





							def 			A__				(    self):
												pass






import numpy as np

import datasets


lowercase__		:Dict			  =   "\nCompute the Mahalanobis Distance\n\nMahalonobis distance is the distance between a point and a distribution.\nAnd not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance.\nIt was introduced by Prof. P. C. Mahalanobis in 1936\nand has been used in various statistical applications ever since\n[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/]\n"

lowercase__		:List[Any]			  =   "\\n@article{de2000mahalanobis,\n  title={The mahalanobis distance},\n  author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L},\n  journal={Chemometrics and intelligent laboratory systems},\n  volume={50},\n  number={1},\n  pages={1--18},\n  year={2000},\n  publisher={Elsevier}\n}\n"

lowercase__		:Dict			  =   "\nArgs:\n    X: List of datapoints to be compared with the `reference_distribution`.\n    reference_distribution: List of datapoints from the reference distribution we want to compare to.\nReturns:\n    mahalanobis: The Mahalonobis distance for each datapoint in `X`.\nExamples:\n\n    >>> mahalanobis_metric = datasets.load_metric(\"mahalanobis\")\n    >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]])\n    >>> print(results)\n    {'mahalanobis': array([0.5])}\n"







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,  _KWARGS_DESCRIPTION		)
class     lowercase		( datasets.Metric		):




							def 			A__				(    self):
												return datasets.MetricInfo(
												    description=_DESCRIPTION       ,citation=_CITATION       ,inputs_description=_KWARGS_DESCRIPTION       ,features=datasets.Features(
												        {
												            '''X''': datasets.Sequence(datasets.Value('''float'''       ,id='''sequence''')       ,id='''X'''),
												        })       ,)





							def 			A__				(    self       ,A__       ,A__):
												# convert to numpy arrays
												lowercase						     =						np.array(A__)
												lowercase						     =						np.array(A__)

												# Assert that arrays are 2D
												if len(X.shape) != 2:
																	raise ValueError('''Expected `X` to be a 2D vector''')
												if len(reference_distribution.shape) != 2:
																	raise ValueError('''Expected `reference_distribution` to be a 2D vector''')
												if reference_distribution.shape[0] < 2:
																	raise ValueError(
																	    '''Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension''')

												# Get mahalanobis distance for each prediction
												lowercase						     =						X - np.mean(A__)
												lowercase						     =						np.cov(reference_distribution.T)
												try:
																	lowercase						     =						np.linalg.inv(A__)
												except np.linalg.LinAlgError:
																	lowercase						     =						np.linalg.pinv(A__)
												lowercase						     =						np.dot(A__       ,A__)
												lowercase						     =						np.dot(A__       ,X_minus_mu.T).diagonal()

												return {"mahalanobis": mahal_dist}




