






'''simple docstring'''







import os

import jsonlines
import numpy as np
from tqdm import tqdm


_A     :      Dict   					=2_048
_A     :      List[str]   					=4_096
_A     :      str   					=42
_A     :      Tuple   					=os.environ.pop('''PROCESS_TRAIN''', '''false''')
_A     :      Any   					={'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase							)				->	Optional[int]:
							def choose_first(UpperCamelCase    ,      UpperCamelCase=False							):
														assert isinstance(UpperCamelCase    ,      UpperCamelCase							)
														if len(UpperCamelCase							) == 1:
																					lowerCamelCase__ :     Any	  =	answer[0]
																					return {k: [answer[k]] for k in answer} if is_long_answer else answer
														for a in answer:
																					if is_long_answer:
																												lowerCamelCase__ :     Optional[int]	  =	{k: [a[k]] for k in a}
																					if len(a["""start_token"""]							) > 0:
																												break
														return a

							lowerCamelCase__ :     Tuple	  =	{"""id""": example["""id"""]}
							lowerCamelCase__ :     Union[str, Any]	  =	example["""annotations"""]
							lowerCamelCase__ :     Optional[Any]	  =	annotation["""yes_no_answer"""]
							if 0 in yes_no_answer or 1 in yes_no_answer:
														lowerCamelCase__ :     str	  =	["""yes"""] if 1 in yes_no_answer else ["""no"""]
														lowerCamelCase__ :     str	  =	[]
														lowerCamelCase__ :     str	  =	[]
														lowerCamelCase__ :     List[str]	  =	["""<cls>"""]
							else:
														lowerCamelCase__ :     Union[str, Any]	  =	["""short"""]
														lowerCamelCase__ :     str	  =	choose_first(annotation["""short_answers"""]							)
														if len(out["""start_token"""]							) == 0:
																					# answer will be long if short is not available
																					lowerCamelCase__ :     Tuple	  =	["""long"""]
																					lowerCamelCase__ :     Optional[int]	  =	choose_first(annotation["""long_answer"""]    ,      is_long_answer=UpperCamelCase							)
																					lowerCamelCase__ :     Optional[int]	  =	[]
														answer.update(UpperCamelCase							)

							# disregard some samples
							if len(answer["""start_token"""]							) > 1 or answer["start_token"] == answer["end_token"]:
														lowerCamelCase__ :     List[str]	  =	True
							else:
														lowerCamelCase__ :     List[str]	  =	False

							lowerCamelCase__ :     List[Any]	  =	["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
							if not all(isinstance(answer[k]    ,      UpperCamelCase							) for k in cols							):
														raise ValueError("""Issue in ID"""    ,      example["""id"""]							)

							return answer





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase=False							)				->	Any:
							lowerCamelCase__ :     List[str]	  =	_get_single_answer(UpperCamelCase							)
							# bytes are of no use
							del answer["start_byte"]
							del answer["end_byte"]

							# handle yes_no answers explicitly
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														lowerCamelCase__ :     List[Any]	  =	example["""document"""]["""tokens"""]
														lowerCamelCase__ :     int	  =	[]
														for i in range(len(doc["""token"""]							)							):
																					if not doc["is_html"][i]:
																												context.append(doc["""token"""][i]							)
														return {
														    "context": " ".join(UpperCamelCase							),
														    "answer": {
														        "start_token": -100,  # ignore index in cross-entropy
														        "end_token": -100,  # ignore index in cross-entropy
														        "category": answer["category"],
														        "span": answer["category"],  # extra
														    },
														}

							# later, help in removing all no answers
							if answer["start_token"] == [-1]:
														return {
														    "context": "None",
														    "answer": {
														        "start_token": -1,
														        "end_token": -1,
														        "category": "null",
														        "span": "None",  # extra
														    },
														}

							# handling normal samples

							lowerCamelCase__ :     Optional[Any]	  =	["""start_token""", """end_token"""]
							answer.update({k: answer[k][0] if len(answer[k]							) > 0 else answer[k] for k in cols}							)  # e.g. [10] == 10

							lowerCamelCase__ :     int	  =	example["""document"""]["""tokens"""]
							lowerCamelCase__ :     Union[str, Any]	  =	answer["""start_token"""]
							lowerCamelCase__ :     Tuple	  =	answer["""end_token"""]

							lowerCamelCase__ :     Dict	  =	[]
							for i in range(len(doc["""token"""]							)							):
														if not doc["is_html"][i]:
																					context.append(doc["""token"""][i]							)
														else:
																					if answer["start_token"] > i:
																												start_token -= 1
																					if answer["end_token"] > i:
																												end_token -= 1
							lowerCamelCase__ :     Union[str, Any]	  =	""" """.join(context[start_token:end_token]							)

							# checking above code
							if assertion:
														lowerCamelCase__ :     List[Any]	  =	doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
														lowerCamelCase__ :     Union[str, Any]	  =	doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
														lowerCamelCase__ :     Tuple	  =	""" """.join([old[i] for i in range(len(UpperCamelCase							)							) if not is_html[i]]							)
														if new != old:
																					print("""ID:"""    ,      example["""id"""]							)
																					print("""New:"""    ,      UpperCamelCase    ,      end="""\n"""							)
																					print("""Old:"""    ,      UpperCamelCase    ,      end="""\n\n"""							)

							return {
							    "context": " ".join(UpperCamelCase							),
							    "answer": {
							        "start_token": start_token,
							        "end_token": end_token - 1,  # this makes it inclusive
							        "category": answer["category"],  # either long or short
							        "span": new,  # extra
							    },
							}





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase    ,      UpperCamelCase=2048    ,      UpperCamelCase=4096    ,      UpperCamelCase=True							)				->	Any:
							# overlap will be of doc_stride - q_len

							lowerCamelCase__ :     Optional[int]	  =	get_context_and_ans(UpperCamelCase    ,      assertion=UpperCamelCase							)
							lowerCamelCase__ :     str	  =	out["""answer"""]

							# later, removing these samples
							if answer["start_token"] == -1:
														return {
														    "example_id": example["id"],
														    "input_ids": [[-1]],
														    "labels": {
														        "start_token": [-1],
														        "end_token": [-1],
														        "category": ["null"],
														    },
														}

							lowerCamelCase__ :     Dict	  =	tokenizer(example["""question"""]["""text"""]    ,      out["""context"""]							).input_ids
							lowerCamelCase__ :     Union[str, Any]	  =	input_ids.index(tokenizer.sep_token_id							) + 1

							# return yes/no
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														lowerCamelCase__ :     Union[str, Any]	  =	[]
														lowerCamelCase__ :     List[Any]	  =	[]
														lowerCamelCase__ :     Union[str, Any]	  =	input_ids[:q_len]
														lowerCamelCase__ :     Tuple	  =	range(UpperCamelCase    ,      len(UpperCamelCase							)    ,      max_length - doc_stride							)
														for i in doc_start_indices:
																					lowerCamelCase__ :     Optional[int]	  =	i + max_length - q_len
																					lowerCamelCase__ :     List[str]	  =	input_ids[i:end_index]
																					inputs.append(q_indices + slice							)
																					category.append(answer["""category"""][0]							)
																					if slice[-1] == tokenizer.sep_token_id:
																												break

														return {
														    "example_id": example["id"],
														    "input_ids": inputs,
														    "labels": {
														        "start_token": [-100] * len(UpperCamelCase							),
														        "end_token": [-100] * len(UpperCamelCase							),
														        "category": category,
														    },
														}

							lowerCamelCase__ :     str	  =	out["""context"""].split()
							lowerCamelCase__ :     List[Any]	  =	splitted_context[answer["""end_token"""]]
							lowerCamelCase__ :     List[Any]	  =	len(
							    tokenizer(
							        """ """.join(splitted_context[: answer["""start_token"""]]							)    ,      add_special_tokens=UpperCamelCase    ,      ).input_ids							)
							lowerCamelCase__ :     Dict	  =	len(
							    tokenizer(""" """.join(splitted_context[: answer["""end_token"""]]							)    ,      add_special_tokens=UpperCamelCase							).input_ids							)

							answer["start_token"] += q_len
							answer["end_token"] += q_len

							# fixing end token
							lowerCamelCase__ :     int	  =	len(tokenizer(UpperCamelCase    ,      add_special_tokens=UpperCamelCase							).input_ids							)
							if num_sub_tokens > 1:
														answer["end_token"] += num_sub_tokens - 1

							lowerCamelCase__ :     Union[str, Any]	  =	input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
							lowerCamelCase__ :     List[Any]	  =	answer["""start_token"""]
							lowerCamelCase__ :     List[Any]	  =	answer["""end_token"""]

							if assertion:
														lowerCamelCase__ :     List[Any]	  =	tokenizer.decode(UpperCamelCase							)
														if answer["span"] != new:
																					print("""ISSUE IN TOKENIZATION"""							)
																					print("""OLD:"""    ,      answer["""span"""]							)
																					print("""NEW:"""    ,      UpperCamelCase    ,      end="""\n\n"""							)

							if len(UpperCamelCase							) <= max_length:
														return {
														    "example_id": example["id"],
														    "input_ids": [input_ids],
														    "labels": {
														        "start_token": [answer["start_token"]],
														        "end_token": [answer["end_token"]],
														        "category": answer["category"],
														    },
														}

							lowerCamelCase__ :     int	  =	input_ids[:q_len]
							lowerCamelCase__ :     Dict	  =	range(UpperCamelCase    ,      len(UpperCamelCase							)    ,      max_length - doc_stride							)

							lowerCamelCase__ :     str	  =	[]
							lowerCamelCase__ :     Optional[int]	  =	[]
							lowerCamelCase__ :     Optional[int]	  =	[]
							lowerCamelCase__ :     List[str]	  =	[]  # null, yes, no, long, short
							for i in doc_start_indices:
														lowerCamelCase__ :     List[str]	  =	i + max_length - q_len
														lowerCamelCase__ :     Optional[Any]	  =	input_ids[i:end_index]
														inputs.append(q_indices + slice							)
														assert len(inputs[-1]							) <= max_length, "Issue in truncating length"

														if start_token >= i and end_token <= end_index - 1:
																					lowerCamelCase__ :     Tuple	  =	start_token - i + q_len
																					lowerCamelCase__ :     int	  =	end_token - i + q_len
																					answers_category.append(answer["""category"""][0]							)  # ["short"] -> "short"
														else:
																					lowerCamelCase__ :     List[str]	  =	-100
																					lowerCamelCase__ :     Tuple	  =	-100
																					answers_category.append("""null"""							)
														lowerCamelCase__ :     Union[str, Any]	  =	inputs[-1][start_token : end_token + 1]

														answers_start_token.append(UpperCamelCase							)
														answers_end_token.append(UpperCamelCase							)
														if assertion:
																					if new != old and new != [tokenizer.cls_token_id]:
																												print("""ISSUE in strided for ID:"""    ,      example["""id"""]							)
																												print("""New:"""    ,      tokenizer.decode(UpperCamelCase							)							)
																												print("""Old:"""    ,      tokenizer.decode(UpperCamelCase							)    ,      end="""\n\n"""							)
														if slice[-1] == tokenizer.sep_token_id:
																					break

							return {
							    "example_id": example["id"],
							    "input_ids": inputs,
							    "labels": {
							        "start_token": answers_start_token,
							        "end_token": answers_end_token,
							        "category": answers_category,
							    },
							}





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase    ,      UpperCamelCase=2048    ,      UpperCamelCase=4096    ,      UpperCamelCase=False							)				->	Dict:
							lowerCamelCase__ :     Optional[Any]	  =	get_strided_contexts_and_ans(
							    UpperCamelCase    ,      UpperCamelCase    ,      doc_stride=UpperCamelCase    ,      max_length=UpperCamelCase    ,      assertion=UpperCamelCase    ,      )

							return example





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase							)				->	Tuple:
							with jsonlines.open(UpperCamelCase    ,      """a"""							) as writer:
														for example in tqdm(UpperCamelCase    ,      total=len(UpperCamelCase							)    ,      desc="""Saving samples ... """							):
																					lowerCamelCase__ :     str	  =	example["""labels"""]
																					for ids, start, end, cat in zip(
																					    example["""input_ids"""]    ,      labels["""start_token"""]    ,      labels["""end_token"""]    ,      labels["""category"""]    ,      ):
																												if start == -1 and end == -1:
																																			continue  # leave waste samples with no answer
																												if cat == "null" and np.random.rand() < 0.6:
																																			continue  # removing 50 % samples
																												writer.write(
																												    {
																												        """input_ids""": ids,
																												        """start_token""": start,
																												        """end_token""": end,
																												        """category""": CATEGORY_MAPPING[cat],
																												    }							)


if __name__ == "__main__":
			from datasets import load_dataset

			from transformers import BigBirdTokenizer

			_A     :      Tuple   					=load_dataset('''natural_questions''')
			_A     :      Optional[int]   					=BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

			_A     :      str   					=data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

			_A     :      int   					={
			    '''tokenizer''': tokenizer,
			    '''doc_stride''': DOC_STRIDE,
			    '''max_length''': MAX_LENGTH,
			    '''assertion''': False,
			}
			_A     :      Tuple   					=data.map(prepare_inputs, fn_kwargs=fn_kwargs)
			_A     :      Any   					=data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
			print(data)

			np.random.seed(SEED)
			_A     :      Union[str, Any]   					='''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
			save_to_disk(data, file_name=cache_file_name)









'''simple docstring'''







from __future__ import annotations

import typing
from collections.abc import Iterable

import numpy as np

_A     :      Optional[Any]   					=typing.Union[Iterable[float], Iterable[int], np.ndarray]  # noqa: UP007
_A     :      Optional[int]   					=typing.Union[np.floataa, int, float]  # noqa: UP007





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase				)				->	VectorOut:
							return np.sqrt(np.sum((np.asarray(UpperCamelCase				) - np.asarray(UpperCamelCase				)) ** 2				)				)





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase    ,      UpperCamelCase				)				->	VectorOut:
							return sum((va - va) ** 2 for va, va in zip(UpperCamelCase    ,      UpperCamelCase				)				) ** (1 / 2)


if __name__ == "__main__":





			def 			SCREAMING_SNAKE_CASE_	()				->	None:
										from timeit import timeit

										print("""Without Numpy"""				)
										print(
										    timeit(
										        """euclidean_distance_no_np([1, 2, 3], [4, 5, 6])"""    ,      number=10000    ,      globals=globals()    ,      )				)
										print("""With Numpy"""				)
										print(
										    timeit(
										        """euclidean_distance([1, 2, 3], [4, 5, 6])"""    ,      number=10000    ,      globals=globals()    ,      )				)



			benchmark()

