





import os

import jsonlines
import numpy as np
from tqdm import tqdm


UpperCamelCase__     =	2_048
UpperCamelCase__     =	4_096
UpperCamelCase__     =	42
UpperCamelCase__     =	os.environ.pop("PROCESS_TRAIN", "false")
UpperCamelCase__     =	{"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}





def       _UpperCamelCase       (a__				:str    ):






							"""simple docstring"""





							def choose_first(a__				:str				,     a__				:List[Any]=False    ):
														assert isinstance(a__				,     a__    )
														if len(a__    ) == 1:
																					UpperCamelCase__			       =     answer[0]
																					return {k: [answer[k]] for k in answer} if is_long_answer else answer
														for a in answer:
																					if is_long_answer:
																												UpperCamelCase__			       =     {k: [a[k]] for k in a}
																					if len(a["""start_token"""]    ) > 0:
																												break
														return a

							UpperCamelCase__			       =     {"""id""": example["""id"""]}
							UpperCamelCase__			       =     example["""annotations"""]
							UpperCamelCase__			       =     annotation["""yes_no_answer"""]
							if 0 in yes_no_answer or 1 in yes_no_answer:
														UpperCamelCase__			       =     ["""yes"""] if 1 in yes_no_answer else ["""no"""]
														UpperCamelCase__			       =     UpperCamelCase__			       =     []
														UpperCamelCase__			       =     UpperCamelCase__			       =     []
														UpperCamelCase__			       =     ["""<cls>"""]
							else:
														UpperCamelCase__			       =     ["""short"""]
														UpperCamelCase__			       =     choose_first(annotation["""short_answers"""]    )
														if len(out["""start_token"""]    ) == 0:
																					# answer will be long if short is not available
																					UpperCamelCase__			       =     ["""long"""]
																					UpperCamelCase__			       =     choose_first(annotation["""long_answer"""]				,     is_long_answer=a__    )
																					UpperCamelCase__			       =     []
														answer.update(a__    )

							# disregard some samples
							if len(answer["""start_token"""]    ) > 1 or answer["start_token"] == answer["end_token"]:
														UpperCamelCase__			       =     True
							else:
														UpperCamelCase__			       =     False

							UpperCamelCase__			       =     ["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
							if not all(isinstance(answer[k]				,     a__    ) for k in cols    ):
														raise ValueError("""Issue in ID"""				,     example["""id"""]    )

							return answer





def       _UpperCamelCase       (a__				:Dict				,     a__				:Union[str, Any]=False    ):






							"""simple docstring"""





							UpperCamelCase__			       =     _get_single_answer(a__    )
							# bytes are of no use
							del answer["start_byte"]
							del answer["end_byte"]

							# handle yes_no answers explicitly
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														UpperCamelCase__			       =     example["""document"""]["""tokens"""]
														UpperCamelCase__			       =     []
														for i in range(len(doc["""token"""]    )    ):
																					if not doc["is_html"][i]:
																												context.append(doc["""token"""][i]    )
														return {
														    "context": " ".join(a__    ),
														    "answer": {
														        "start_token": -100,  # ignore index in cross-entropy
														        "end_token": -100,  # ignore index in cross-entropy
														        "category": answer["category"],
														        "span": answer["category"],  # extra
														    },
														}

							# later, help in removing all no answers
							if answer["start_token"] == [-1]:
														return {
														    "context": "None",
														    "answer": {
														        "start_token": -1,
														        "end_token": -1,
														        "category": "null",
														        "span": "None",  # extra
														    },
														}

							# handling normal samples

							UpperCamelCase__			       =     ["""start_token""", """end_token"""]
							answer.update({k: answer[k][0] if len(answer[k]    ) > 0 else answer[k] for k in cols}    )  # e.g. [10] == 10

							UpperCamelCase__			       =     example["""document"""]["""tokens"""]
							UpperCamelCase__			       =     answer["""start_token"""]
							UpperCamelCase__			       =     answer["""end_token"""]

							UpperCamelCase__			       =     []
							for i in range(len(doc["""token"""]    )    ):
														if not doc["is_html"][i]:
																					context.append(doc["""token"""][i]    )
														else:
																					if answer["start_token"] > i:
																												start_token -= 1
																					if answer["end_token"] > i:
																												end_token -= 1
							UpperCamelCase__			       =     """ """.join(context[start_token:end_token]    )

							# checking above code
							if assertion:
														UpperCamelCase__			       =     doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
														UpperCamelCase__			       =     doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
														UpperCamelCase__			       =     """ """.join([old[i] for i in range(len(a__    )    ) if not is_html[i]]    )
														if new != old:
																					print("""ID:"""				,     example["""id"""]    )
																					print("""New:"""				,     a__				,     end="""\n"""    )
																					print("""Old:"""				,     a__				,     end="""\n\n"""    )

							return {
							    "context": " ".join(a__    ),
							    "answer": {
							        "start_token": start_token,
							        "end_token": end_token - 1,  # this makes it inclusive
							        "category": answer["category"],  # either long or short
							        "span": new,  # extra
							    },
							}





def       _UpperCamelCase       (a__				:Optional[int]				,     a__				:Optional[Any]				,     a__				:Union[str, Any]=2048				,     a__				:Optional[int]=4096				,     a__				:Optional[Any]=True    ):






							"""simple docstring"""





							UpperCamelCase__			       =     get_context_and_ans(a__				,     assertion=a__    )
							UpperCamelCase__			       =     out["""answer"""]

							# later, removing these samples
							if answer["start_token"] == -1:
														return {
														    "example_id": example["id"],
														    "input_ids": [[-1]],
														    "labels": {
														        "start_token": [-1],
														        "end_token": [-1],
														        "category": ["null"],
														    },
														}

							UpperCamelCase__			       =     tokenizer(example["""question"""]["""text"""]				,     out["""context"""]    ).input_ids
							UpperCamelCase__			       =     input_ids.index(tokenizer.sep_token_id    ) + 1

							# return yes/no
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														UpperCamelCase__			       =     []
														UpperCamelCase__			       =     []
														UpperCamelCase__			       =     input_ids[:q_len]
														UpperCamelCase__			       =     range(a__				,     len(a__    )				,     max_length - doc_stride    )
														for i in doc_start_indices:
																					UpperCamelCase__			       =     i + max_length - q_len
																					UpperCamelCase__			       =     input_ids[i:end_index]
																					inputs.append(q_indices + slice    )
																					category.append(answer["""category"""][0]    )
																					if slice[-1] == tokenizer.sep_token_id:
																												break

														return {
														    "example_id": example["id"],
														    "input_ids": inputs,
														    "labels": {
														        "start_token": [-100] * len(a__    ),
														        "end_token": [-100] * len(a__    ),
														        "category": category,
														    },
														}

							UpperCamelCase__			       =     out["""context"""].split()
							UpperCamelCase__			       =     splitted_context[answer["""end_token"""]]
							UpperCamelCase__			       =     len(
							    tokenizer(
							        """ """.join(splitted_context[: answer["""start_token"""]]    )				,     add_special_tokens=a__				,     ).input_ids    )
							UpperCamelCase__			       =     len(
							    tokenizer(""" """.join(splitted_context[: answer["""end_token"""]]    )				,     add_special_tokens=a__    ).input_ids    )

							answer["start_token"] += q_len
							answer["end_token"] += q_len

							# fixing end token
							UpperCamelCase__			       =     len(tokenizer(a__				,     add_special_tokens=a__    ).input_ids    )
							if num_sub_tokens > 1:
														answer["end_token"] += num_sub_tokens - 1

							UpperCamelCase__			       =     input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
							UpperCamelCase__			       =     answer["""start_token"""]
							UpperCamelCase__			       =     answer["""end_token"""]

							if assertion:
														UpperCamelCase__			       =     tokenizer.decode(a__    )
														if answer["span"] != new:
																					print("""ISSUE IN TOKENIZATION"""    )
																					print("""OLD:"""				,     answer["""span"""]    )
																					print("""NEW:"""				,     a__				,     end="""\n\n"""    )

							if len(a__    ) <= max_length:
														return {
														    "example_id": example["id"],
														    "input_ids": [input_ids],
														    "labels": {
														        "start_token": [answer["start_token"]],
														        "end_token": [answer["end_token"]],
														        "category": answer["category"],
														    },
														}

							UpperCamelCase__			       =     input_ids[:q_len]
							UpperCamelCase__			       =     range(a__				,     len(a__    )				,     max_length - doc_stride    )

							UpperCamelCase__			       =     []
							UpperCamelCase__			       =     []
							UpperCamelCase__			       =     []
							UpperCamelCase__			       =     []  # null, yes, no, long, short
							for i in doc_start_indices:
														UpperCamelCase__			       =     i + max_length - q_len
														UpperCamelCase__			       =     input_ids[i:end_index]
														inputs.append(q_indices + slice    )
														assert len(inputs[-1]    ) <= max_length, "Issue in truncating length"

														if start_token >= i and end_token <= end_index - 1:
																					UpperCamelCase__			       =     start_token - i + q_len
																					UpperCamelCase__			       =     end_token - i + q_len
																					answers_category.append(answer["""category"""][0]    )  # ["short"] -> "short"
														else:
																					UpperCamelCase__			       =     -100
																					UpperCamelCase__			       =     -100
																					answers_category.append("""null"""    )
														UpperCamelCase__			       =     inputs[-1][start_token : end_token + 1]

														answers_start_token.append(a__    )
														answers_end_token.append(a__    )
														if assertion:
																					if new != old and new != [tokenizer.cls_token_id]:
																												print("""ISSUE in strided for ID:"""				,     example["""id"""]    )
																												print("""New:"""				,     tokenizer.decode(a__    )    )
																												print("""Old:"""				,     tokenizer.decode(a__    )				,     end="""\n\n"""    )
														if slice[-1] == tokenizer.sep_token_id:
																					break

							return {
							    "example_id": example["id"],
							    "input_ids": inputs,
							    "labels": {
							        "start_token": answers_start_token,
							        "end_token": answers_end_token,
							        "category": answers_category,
							    },
							}





def       _UpperCamelCase       (a__				:Optional[Any]				,     a__				:Tuple				,     a__				:Optional[Any]=2048				,     a__				:Tuple=4096				,     a__				:Optional[Any]=False    ):






							"""simple docstring"""





							UpperCamelCase__			       =     get_strided_contexts_and_ans(
							    a__				,     a__				,     doc_stride=a__				,     max_length=a__				,     assertion=a__				,     )

							return example





def       _UpperCamelCase       (a__				:Optional[int]				,     a__				:Optional[int]    ):






							"""simple docstring"""





							with jsonlines.open(a__				,     """a"""    ) as writer:
														for example in tqdm(a__				,     total=len(a__    )				,     desc="""Saving samples ... """    ):
																					UpperCamelCase__			       =     example["""labels"""]
																					for ids, start, end, cat in zip(
																					    example["""input_ids"""]				,     labels["""start_token"""]				,     labels["""end_token"""]				,     labels["""category"""]				,     ):
																												if start == -1 and end == -1:
																																			continue  # leave waste samples with no answer
																												if cat == "null" and np.random.rand() < 0.6:
																																			continue  # removing 50 % samples
																												writer.write(
																												    {
																												        """input_ids""": ids,
																												        """start_token""": start,
																												        """end_token""": end,
																												        """category""": CATEGORY_MAPPING[cat],
																												    }    )


if __name__ == "__main__":
						from datasets import load_dataset

						from transformers import BigBirdTokenizer

						UpperCamelCase__     =	load_dataset("natural_questions")
						UpperCamelCase__     =	BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

						UpperCamelCase__     =	data["train" if PROCESS_TRAIN == "true" else "validation"]

						UpperCamelCase__     =	{
						    "tokenizer": tokenizer,
						    "doc_stride": DOC_STRIDE,
						    "max_length": MAX_LENGTH,
						    "assertion": False,
						}
						UpperCamelCase__     =	data.map(prepare_inputs, fn_kwargs=fn_kwargs)
						UpperCamelCase__     =	data.remove_columns(["annotations", "document", "id", "question"])
						print(data)

						np.random.seed(SEED)
						UpperCamelCase__     =	"nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
						save_to_disk(data, file_name=cache_file_name)











from datetime import datetime as dt
import os

from github import Github


UpperCamelCase__     =	[
    "good first issue",
    "good second issue",
    "good difficult issue",
    "feature request",
    "new model",
    "wip",
]





def       _UpperCamelCase       ():






							"""simple docstring"""





							UpperCamelCase__			       =     Github(os.environ["""GITHUB_TOKEN"""]    )
							UpperCamelCase__			       =     g.get_repo("""huggingface/transformers"""    )
							UpperCamelCase__			       =     repo.get_issues(state="""open"""    )

							for issue in open_issues:
														UpperCamelCase__			       =     sorted([comment for comment in issue.get_comments()]				,     key=lambda a__    : i.created_at				,     reverse=a__    )
														UpperCamelCase__			       =     comments[0] if len(a__    ) > 0 else None
														if (
														    last_comment is not None
														    and last_comment.user.login == "github-actions[bot]"
														    and (dt.utcnow() - issue.updated_at).days > 7
														    and (dt.utcnow() - issue.created_at).days >= 30
														    and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()    )
														):
																					# print(f"Would close issue {issue.number} since it has been 7 days of inactivity since bot mention.")
																					issue.edit(state="""closed"""    )
														elif (
														    (dt.utcnow() - issue.updated_at).days > 23
														    and (dt.utcnow() - issue.created_at).days >= 30
														    and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()    )
														):
																					# print(f"Would add stale comment to {issue.number}")
																					issue.create_comment(
																					    """This issue has been automatically marked as stale because it has not had """
																					    """recent activity. If you think this still needs to be addressed """
																					    """please comment on this thread.\n\nPlease note that issues that do not follow the """
																					    """[contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) """
																					    """are likely to be ignored."""    )


if __name__ == "__main__":
					main()




