




'''simple docstring'''





import os

import jsonlines
import numpy as np
from tqdm import tqdm


__a						:      List[Any]       =						2048
__a						:      List[str]       =						4096
__a						:      Optional[int]       =						42
__a						:      Optional[Any]       =						os.environ.pop('''PROCESS_TRAIN''', '''false''')
__a						:      Optional[int]       =						{'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}





def      __a     (       _UpperCamelCase:   Union[str, Any]   )						->			Any:

     """simple docstring"""




     def choose_first(_UpperCamelCase:   Optional[int] ,						_UpperCamelCase:   Dict=False   ):
          assert isinstance(_lowerCAmelCase ,						_lowerCAmelCase   )
          if len(_lowerCAmelCase   ) == 1:
               _snake_case							=				answer[0]
               return {k: [answer[k]] for k in answer} if is_long_answer else answer
          for a in answer:
               if is_long_answer:
                    _snake_case							=				{k: [a[k]] for k in a}
               if len(a["start_token"]   ) > 0:
                    break
          return a

     _snake_case							=				{"""id""": example["""id"""]}
     _snake_case							=				example["""annotations"""]
     _snake_case							=				annotation["""yes_no_answer"""]
     if 0 in yes_no_answer or 1 in yes_no_answer:
          _snake_case							=				["""yes"""] if 1 in yes_no_answer else ["""no"""]
          _snake_case							=				[]
          _snake_case							=				[]
          _snake_case							=				["""<cls>"""]
     else:
          _snake_case							=				["""short"""]
          _snake_case							=				choose_first(annotation["short_answers"]   )
          if len(out["start_token"]   ) == 0:
               # answer will be long if short is not available
               _snake_case							=				["""long"""]
               _snake_case							=				choose_first(annotation["long_answer"] ,						is_long_answer=_lowerCAmelCase   )
               _snake_case							=				[]
          answer.update(_lowerCAmelCase   )

     # disregard some samples
     if len(answer["start_token"]   ) > 1 or answer["start_token"] == answer["end_token"]:
          _snake_case							=				True
     else:
          _snake_case							=				False

     _snake_case							=				["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
     if not all(isinstance(answer[k] ,						_lowerCAmelCase   ) for k in cols   ):
          raise ValueError("Issue in ID" ,						example["id"]   )

     return answer





def      __a     (       _UpperCamelCase:   Union[str, Any] ,						_UpperCamelCase:   Union[str, Any]=False   )						->			List[Any]:

     """simple docstring"""




     _snake_case							=				_get_single_answer(_lowerCAmelCase   )
     # bytes are of no use
     del answer["start_byte"]
     del answer["end_byte"]

     # handle yes_no answers explicitly
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _snake_case							=				example["""document"""]["""tokens"""]
          _snake_case							=				[]
          for i in range(len(doc["token"]   )   ):
               if not doc["is_html"][i]:
                    context.append(doc["token"][i]   )
          return {
              "context": " ".join(_lowerCAmelCase   ),
              "answer": {
                  "start_token": -100,  # ignore index in cross-entropy
                  "end_token": -100,  # ignore index in cross-entropy
                  "category": answer["category"],
                  "span": answer["category"],  # extra
              },
          }

     # later, help in removing all no answers
     if answer["start_token"] == [-1]:
          return {
              "context": "None",
              "answer": {
                  "start_token": -1,
                  "end_token": -1,
                  "category": "null",
                  "span": "None",  # extra
              },
          }

     # handling normal samples

     _snake_case							=				["""start_token""", """end_token"""]
     answer.update({k: answer[k][0] if len(answer[k]   ) > 0 else answer[k] for k in cols}   )  # e.g. [10] == 10

     _snake_case							=				example["""document"""]["""tokens"""]
     _snake_case							=				answer["""start_token"""]
     _snake_case							=				answer["""end_token"""]

     _snake_case							=				[]
     for i in range(len(doc["token"]   )   ):
          if not doc["is_html"][i]:
               context.append(doc["token"][i]   )
          else:
               if answer["start_token"] > i:
                    start_token -= 1
               if answer["end_token"] > i:
                    end_token -= 1
     _snake_case							=				""" """.join(context[start_token:end_token]   )

     # checking above code
     if assertion:
          _snake_case							=				doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
          _snake_case							=				doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
          _snake_case							=				""" """.join([old[i] for i in range(len(_lowerCAmelCase   )   ) if not is_html[i]]   )
          if new != old:
               print("ID:" ,						example["id"]   )
               print("New:" ,						_lowerCAmelCase ,						end="\n"   )
               print("Old:" ,						_lowerCAmelCase ,						end="\n\n"   )

     return {
         "context": " ".join(_lowerCAmelCase   ),
         "answer": {
             "start_token": start_token,
             "end_token": end_token - 1,  # this makes it inclusive
             "category": answer["category"],  # either long or short
             "span": new,  # extra
         },
     }





def      __a     (       _UpperCamelCase:   int ,						_UpperCamelCase:   str ,						_UpperCamelCase:   List[Any]=2_048 ,						_UpperCamelCase:   Union[str, Any]=4_096 ,						_UpperCamelCase:   Tuple=True   )						->			Any:

     """simple docstring"""




     _snake_case							=				get_context_and_ans(_lowerCAmelCase ,						assertion=_lowerCAmelCase   )
     _snake_case							=				out["""answer"""]

     # later, removing these samples
     if answer["start_token"] == -1:
          return {
              "example_id": example["id"],
              "input_ids": [[-1]],
              "labels": {
                  "start_token": [-1],
                  "end_token": [-1],
                  "category": ["null"],
              },
          }

     _snake_case							=				tokenizer(example["question"]["text"] ,						out["context"]   ).input_ids
     _snake_case							=				input_ids.index(tokenizer.sep_token_id   ) + 1

     # return yes/no
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _snake_case							=				[]
          _snake_case							=				[]
          _snake_case							=				input_ids[:q_len]
          _snake_case							=				range(_lowerCAmelCase ,						len(_lowerCAmelCase   ) ,						max_length - doc_stride   )
          for i in doc_start_indices:
               _snake_case							=				i + max_length - q_len
               _snake_case							=				input_ids[i:end_index]
               inputs.append(q_indices + slice   )
               category.append(answer["category"][0]   )
               if slice[-1] == tokenizer.sep_token_id:
                    break

          return {
              "example_id": example["id"],
              "input_ids": inputs,
              "labels": {
                  "start_token": [-100] * len(_lowerCAmelCase   ),
                  "end_token": [-100] * len(_lowerCAmelCase   ),
                  "category": category,
              },
          }

     _snake_case							=				out["""context"""].split()
     _snake_case							=				splitted_context[answer["""end_token"""]]
     _snake_case							=				len(
         tokenizer(
             " ".join(splitted_context[: answer["start_token"]]   ) ,						add_special_tokens=_lowerCAmelCase ,						).input_ids   )
     _snake_case							=				len(
         tokenizer(" ".join(splitted_context[: answer["end_token"]]   ) ,						add_special_tokens=_lowerCAmelCase   ).input_ids   )

     answer["start_token"] += q_len
     answer["end_token"] += q_len

     # fixing end token
     _snake_case							=				len(tokenizer(_lowerCAmelCase ,						add_special_tokens=_lowerCAmelCase   ).input_ids   )
     if num_sub_tokens > 1:
          answer["end_token"] += num_sub_tokens - 1

     _snake_case							=				input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
     _snake_case							=				answer["""start_token"""]
     _snake_case							=				answer["""end_token"""]

     if assertion:
          _snake_case							=				tokenizer.decode(_lowerCAmelCase   )
          if answer["span"] != new:
               print("ISSUE IN TOKENIZATION"   )
               print("OLD:" ,						answer["span"]   )
               print("NEW:" ,						_lowerCAmelCase ,						end="\n\n"   )

     if len(_lowerCAmelCase   ) <= max_length:
          return {
              "example_id": example["id"],
              "input_ids": [input_ids],
              "labels": {
                  "start_token": [answer["start_token"]],
                  "end_token": [answer["end_token"]],
                  "category": answer["category"],
              },
          }

     _snake_case							=				input_ids[:q_len]
     _snake_case							=				range(_lowerCAmelCase ,						len(_lowerCAmelCase   ) ,						max_length - doc_stride   )

     _snake_case							=				[]
     _snake_case							=				[]
     _snake_case							=				[]
     _snake_case							=				[]  # null, yes, no, long, short
     for i in doc_start_indices:
          _snake_case							=				i + max_length - q_len
          _snake_case							=				input_ids[i:end_index]
          inputs.append(q_indices + slice   )
          assert len(inputs[-1]   ) <= max_length, "Issue in truncating length"

          if start_token >= i and end_token <= end_index - 1:
               _snake_case							=				start_token - i + q_len
               _snake_case							=				end_token - i + q_len
               answers_category.append(answer["category"][0]   )  # ["short"] -> "short"
          else:
               _snake_case							=				-100
               _snake_case							=				-100
               answers_category.append("null"   )
          _snake_case							=				inputs[-1][start_token : end_token + 1]

          answers_start_token.append(_lowerCAmelCase   )
          answers_end_token.append(_lowerCAmelCase   )
          if assertion:
               if new != old and new != [tokenizer.cls_token_id]:
                    print("ISSUE in strided for ID:" ,						example["id"]   )
                    print("New:" ,						tokenizer.decode(_lowerCAmelCase   )   )
                    print("Old:" ,						tokenizer.decode(_lowerCAmelCase   ) ,						end="\n\n"   )
          if slice[-1] == tokenizer.sep_token_id:
               break

     return {
         "example_id": example["id"],
         "input_ids": inputs,
         "labels": {
             "start_token": answers_start_token,
             "end_token": answers_end_token,
             "category": answers_category,
         },
     }





def      __a     (       _UpperCamelCase:   str ,						_UpperCamelCase:   Optional[Any] ,						_UpperCamelCase:   List[Any]=2_048 ,						_UpperCamelCase:   Dict=4_096 ,						_UpperCamelCase:   Optional[Any]=False   )						->			Dict:

     """simple docstring"""




     _snake_case							=				get_strided_contexts_and_ans(
         _lowerCAmelCase ,						_lowerCAmelCase ,						doc_stride=_lowerCAmelCase ,						max_length=_lowerCAmelCase ,						assertion=_lowerCAmelCase ,						)

     return example





def      __a     (       _UpperCamelCase:   Any ,						_UpperCamelCase:   List[str]   )						->			List[Any]:

     """simple docstring"""




     with jsonlines.open(_lowerCAmelCase ,						"a"   ) as writer:
          for example in tqdm(_lowerCAmelCase ,						total=len(_lowerCAmelCase   ) ,						desc="Saving samples ... "   ):
               _snake_case							=				example["""labels"""]
               for ids, start, end, cat in zip(
                   example["input_ids"] ,						labels["start_token"] ,						labels["end_token"] ,						labels["category"] ,						):
                    if start == -1 and end == -1:
                         continue  # leave waste samples with no answer
                    if cat == "null" and np.random.rand() < 0.6:
                         continue  # removing 50 % samples
                    writer.write(
                        {
                            "input_ids": ids,
                            "start_token": start,
                            "end_token": end,
                            "category": CATEGORY_MAPPING[cat],
                        }   )


if __name__ == "__main__":
   from datasets import load_dataset

   from transformers import BigBirdTokenizer

   __a						:      Union[str, Any]       =						load_dataset('''natural_questions''')
   __a						:      List[str]       =						BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

   __a						:      List[Any]       =						data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

   __a						:      List[str]       =						{
       '''tokenizer''': tokenizer,
       '''doc_stride''': DOC_STRIDE,
       '''max_length''': MAX_LENGTH,
       '''assertion''': False,
   }
   __a						:      Optional[Any]       =						data.map(prepare_inputs, fn_kwargs=fn_kwargs)
   __a						:      List[str]       =						data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
   print(data)

   np.random.seed(SEED)
   __a						:      Optional[Any]       =						'''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
   save_to_disk(data, file_name=cache_file_name)








'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_						:      str       =						logging.get_logger(__name__)

UpperCamelCase_						:      Optional[Any]       =						{
    '''asapp/sew-tiny-100k''': '''https://huggingface.co/asapp/sew-tiny-100k/resolve/main/config.json''',
    # See all SEW models at https://huggingface.co/models?filter=sew
}

class        _a    (						__lowerCAmelCase							):
     SCREAMING_SNAKE_CASE_							:   Dict       =						"""sew"""






     def __init__(			self ,_SCREAMING_SNAKE_CASE=32 ,_SCREAMING_SNAKE_CASE=768 ,_SCREAMING_SNAKE_CASE=12 ,_SCREAMING_SNAKE_CASE=12 ,_SCREAMING_SNAKE_CASE=3_072 ,_SCREAMING_SNAKE_CASE=2 ,_SCREAMING_SNAKE_CASE="gelu" ,_SCREAMING_SNAKE_CASE=0.1 ,_SCREAMING_SNAKE_CASE=0.1 ,_SCREAMING_SNAKE_CASE=0.1 ,_SCREAMING_SNAKE_CASE=0.0 ,_SCREAMING_SNAKE_CASE=0.1 ,_SCREAMING_SNAKE_CASE=0.1 ,_SCREAMING_SNAKE_CASE=0.0_2 ,_SCREAMING_SNAKE_CASE=1e-5 ,_SCREAMING_SNAKE_CASE="group" ,_SCREAMING_SNAKE_CASE="gelu" ,_SCREAMING_SNAKE_CASE=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512) ,_SCREAMING_SNAKE_CASE=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1) ,_SCREAMING_SNAKE_CASE=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1) ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=128 ,_SCREAMING_SNAKE_CASE=16 ,_SCREAMING_SNAKE_CASE=True ,_SCREAMING_SNAKE_CASE=0.0_5 ,_SCREAMING_SNAKE_CASE=10 ,_SCREAMING_SNAKE_CASE=2 ,_SCREAMING_SNAKE_CASE=0.0 ,_SCREAMING_SNAKE_CASE=10 ,_SCREAMING_SNAKE_CASE=0 ,_SCREAMING_SNAKE_CASE="mean" ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=256 ,_SCREAMING_SNAKE_CASE=0 ,_SCREAMING_SNAKE_CASE=1 ,_SCREAMING_SNAKE_CASE=2 ,**_SCREAMING_SNAKE_CASE ,)							->      str:
          super().__init__(**_SCREAMING_SNAKE_CASE ,pad_token_id=_SCREAMING_SNAKE_CASE ,bos_token_id=_SCREAMING_SNAKE_CASE ,eos_token_id=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				hidden_size
          _snake_case							=				feat_extract_norm
          _snake_case							=				feat_extract_activation
          _snake_case							=				list(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				list(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				list(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				conv_bias
          _snake_case							=				num_conv_pos_embeddings
          _snake_case							=				num_conv_pos_embedding_groups
          _snake_case							=				len(self.conv_dim   )
          _snake_case							=				num_hidden_layers
          _snake_case							=				intermediate_size
          _snake_case							=				squeeze_factor
          _snake_case							=				hidden_act
          _snake_case							=				num_attention_heads
          _snake_case							=				hidden_dropout
          _snake_case							=				attention_dropout
          _snake_case							=				activation_dropout
          _snake_case							=				feat_proj_dropout
          _snake_case							=				final_dropout
          _snake_case							=				layerdrop
          _snake_case							=				layer_norm_eps
          _snake_case							=				initializer_range
          _snake_case							=				vocab_size

          if (
              (len(self.conv_stride   ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel   ) != self.num_feat_extract_layers)
              or (len(self.conv_dim   ) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   "Configuration for convolutional layers is incorrect."
                   "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"
                   f"""but is `len(config.conv_dim) = {len(self.conv_dim   )}`, `len(config.conv_stride)"""
                   f"""= {len(self.conv_stride   )}`, `len(config.conv_kernel) = {len(self.conv_kernel   )}`."""   )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          _snake_case							=				apply_spec_augment
          _snake_case							=				mask_time_prob
          _snake_case							=				mask_time_length
          _snake_case							=				mask_time_min_masks
          _snake_case							=				mask_feature_prob
          _snake_case							=				mask_feature_length
          _snake_case							=				mask_feature_min_masks

          # ctc loss
          _snake_case							=				ctc_loss_reduction
          _snake_case							=				ctc_zero_infinity

          # sequence classification
          _snake_case							=				use_weighted_layer_sum
          _snake_case							=				classifier_proj_size






     @property
     def     _lowercase     (			self   )							->      Optional[Any]:
          return functools.reduce(operator.mul ,self.conv_stride ,1   )


