"""simple docstring"""




import os

import jsonlines
import numpy as np
from tqdm import tqdm


_a				:		Any	   =       2_048
_a				:		Dict	   =       4_096
_a				:		Tuple	   =       42
_a				:		Optional[Any]	   =       os.environ.pop('PROCESS_TRAIN', 'false')
_a				:		Tuple	   =       {'null': 0, 'short': 1, 'long': 2, 'yes': 3, 'no': 4}





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Union[str, Any]     )	->				List[Any]:
  def choose_first(_lowerCamelCase						:	Union[str, Any]      ,_lowerCamelCase						:	Tuple=False     ):
    assert isinstance(_lowerCamelCase      ,_lowerCamelCase     )
    if len(_lowerCamelCase     ) == 1:
      _lowerCAmelCase						:	List[str]        =					answer[0]
      return {k: [answer[k]] for k in answer} if is_long_answer else answer
    for a in answer:
      if is_long_answer:
        _lowerCAmelCase						:	List[Any]        =					{k: [a[k]] for k in a}
      if len(a["""start_token"""]     ) > 0:
        break
    return a

  _lowerCAmelCase						:	int        =					{"""id""": example["""id"""]}
  _lowerCAmelCase						:	Optional[int]        =					example["""annotations"""]
  _lowerCAmelCase						:	Optional[Any]        =					annotation["""yes_no_answer"""]
  if 0 in yes_no_answer or 1 in yes_no_answer:
    _lowerCAmelCase						:	Tuple        =					["""yes"""] if 1 in yes_no_answer else ["""no"""]
    _lowerCAmelCase						:	Optional[int]        =					[]
    _lowerCAmelCase						:	Optional[Any]        =					[]
    _lowerCAmelCase						:	Any        =					["""<cls>"""]
  else:
    _lowerCAmelCase						:	Dict        =					["""short"""]
    _lowerCAmelCase						:	Dict        =					choose_first(annotation["""short_answers"""]     )
    if len(out["""start_token"""]     ) == 0:
      # answer will be long if short is not available
      _lowerCAmelCase						:	int        =					["""long"""]
      _lowerCAmelCase						:	Optional[Any]        =					choose_first(annotation["""long_answer"""]      ,is_long_answer=_lowerCamelCase     )
      _lowerCAmelCase						:	List[str]        =					[]
    answer.update(_lowerCamelCase     )

  # disregard some samples
  if len(answer["""start_token"""]     ) > 1 or answer["start_token"] == answer["end_token"]:
    _lowerCAmelCase						:	Optional[int]        =					True
  else:
    _lowerCAmelCase						:	int        =					False

  _lowerCAmelCase						:	Tuple        =					["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
  if not all(isinstance(answer[k]      ,_lowerCamelCase     ) for k in cols     ):
    raise ValueError("""Issue in ID"""      ,example["""id"""]     )

  return answer





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Optional[Any]      ,_lowerCamelCase						:	Any=False     )	->				List[Any]:
  _lowerCAmelCase						:	Tuple        =					_get_single_answer(_lowerCamelCase     )
  # bytes are of no use
  del answer["start_byte"]
  del answer["end_byte"]

  # handle yes_no answers explicitly
  if answer["category"][0] in ["yes", "no"]:  # category is list with one element
    _lowerCAmelCase						:	List[Any]        =					example["""document"""]["""tokens"""]
    _lowerCAmelCase						:	Tuple        =					[]
    for i in range(len(doc["""token"""]     )     ):
      if not doc["is_html"][i]:
        context.append(doc["""token"""][i]     )
    return {
        "context": " ".join(_lowerCamelCase     ),
        "answer": {
            "start_token": -100,  # ignore index in cross-entropy
            "end_token": -100,  # ignore index in cross-entropy
            "category": answer["category"],
            "span": answer["category"],  # extra
        },
    }

  # later, help in removing all no answers
  if answer["start_token"] == [-1]:
    return {
        "context": "None",
        "answer": {
            "start_token": -1,
            "end_token": -1,
            "category": "null",
            "span": "None",  # extra
        },
    }

  # handling normal samples

  _lowerCAmelCase						:	Dict        =					["""start_token""", """end_token"""]
  answer.update({k: answer[k][0] if len(answer[k]     ) > 0 else answer[k] for k in cols}     )  # e.g. [10] == 10

  _lowerCAmelCase						:	Dict        =					example["""document"""]["""tokens"""]
  _lowerCAmelCase						:	Optional[Any]        =					answer["""start_token"""]
  _lowerCAmelCase						:	Any        =					answer["""end_token"""]

  _lowerCAmelCase						:	Any        =					[]
  for i in range(len(doc["""token"""]     )     ):
    if not doc["is_html"][i]:
      context.append(doc["""token"""][i]     )
    else:
      if answer["start_token"] > i:
        start_token -= 1
      if answer["end_token"] > i:
        end_token -= 1
  _lowerCAmelCase						:	int        =					""" """.join(context[start_token:end_token]     )

  # checking above code
  if assertion:
    _lowerCAmelCase						:	int        =					doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
    _lowerCAmelCase						:	Tuple        =					doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
    _lowerCAmelCase						:	Optional[int]        =					""" """.join([old[i] for i in range(len(_lowerCamelCase     )     ) if not is_html[i]]     )
    if new != old:
      print("""ID:"""      ,example["""id"""]     )
      print("""New:"""      ,_lowerCamelCase      ,end="""\n"""     )
      print("""Old:"""      ,_lowerCamelCase      ,end="""\n\n"""     )

  return {
      "context": " ".join(_lowerCamelCase     ),
      "answer": {
          "start_token": start_token,
          "end_token": end_token - 1,  # this makes it inclusive
          "category": answer["category"],  # either long or short
          "span": new,  # extra
      },
  }





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Union[str, Any]      ,_lowerCamelCase						:	Any      ,_lowerCamelCase						:	Any=2048      ,_lowerCamelCase						:	List[Any]=4096      ,_lowerCamelCase						:	List[Any]=True     )	->				Optional[int]:
  # overlap will be of doc_stride - q_len

  _lowerCAmelCase						:	str        =					get_context_and_ans(_lowerCamelCase      ,assertion=_lowerCamelCase     )
  _lowerCAmelCase						:	List[Any]        =					out["""answer"""]

  # later, removing these samples
  if answer["start_token"] == -1:
    return {
        "example_id": example["id"],
        "input_ids": [[-1]],
        "labels": {
            "start_token": [-1],
            "end_token": [-1],
            "category": ["null"],
        },
    }

  _lowerCAmelCase						:	List[str]        =					tokenizer(example["""question"""]["""text"""]      ,out["""context"""]     ).input_ids
  _lowerCAmelCase						:	List[Any]        =					input_ids.index(tokenizer.sep_token_id     ) + 1

  # return yes/no
  if answer["category"][0] in ["yes", "no"]:  # category is list with one element
    _lowerCAmelCase						:	Union[str, Any]        =					[]
    _lowerCAmelCase						:	List[Any]        =					[]
    _lowerCAmelCase						:	Union[str, Any]        =					input_ids[:q_len]
    _lowerCAmelCase						:	Union[str, Any]        =					range(_lowerCamelCase      ,len(_lowerCamelCase     )      ,max_length - doc_stride     )
    for i in doc_start_indices:
      _lowerCAmelCase						:	Optional[int]        =					i + max_length - q_len
      _lowerCAmelCase						:	str        =					input_ids[i:end_index]
      inputs.append(q_indices + slice     )
      category.append(answer["""category"""][0]     )
      if slice[-1] == tokenizer.sep_token_id:
        break

    return {
        "example_id": example["id"],
        "input_ids": inputs,
        "labels": {
            "start_token": [-100] * len(_lowerCamelCase     ),
            "end_token": [-100] * len(_lowerCamelCase     ),
            "category": category,
        },
    }

  _lowerCAmelCase						:	str        =					out["""context"""].split()
  _lowerCAmelCase						:	Optional[int]        =					splitted_context[answer["""end_token"""]]
  _lowerCAmelCase						:	Optional[Any]        =					len(
      tokenizer(
          """ """.join(splitted_context[: answer["""start_token"""]]     )      ,add_special_tokens=_lowerCamelCase      ,).input_ids     )
  _lowerCAmelCase						:	str        =					len(
      tokenizer(""" """.join(splitted_context[: answer["""end_token"""]]     )      ,add_special_tokens=_lowerCamelCase     ).input_ids     )

  answer["start_token"] += q_len
  answer["end_token"] += q_len

  # fixing end token
  _lowerCAmelCase						:	List[Any]        =					len(tokenizer(_lowerCamelCase      ,add_special_tokens=_lowerCamelCase     ).input_ids     )
  if num_sub_tokens > 1:
    answer["end_token"] += num_sub_tokens - 1

  _lowerCAmelCase						:	Any        =					input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
  _lowerCAmelCase						:	Optional[Any]        =					answer["""start_token"""]
  _lowerCAmelCase						:	Any        =					answer["""end_token"""]

  if assertion:
    _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.decode(_lowerCamelCase     )
    if answer["span"] != new:
      print("""ISSUE IN TOKENIZATION"""     )
      print("""OLD:"""      ,answer["""span"""]     )
      print("""NEW:"""      ,_lowerCamelCase      ,end="""\n\n"""     )

  if len(_lowerCamelCase     ) <= max_length:
    return {
        "example_id": example["id"],
        "input_ids": [input_ids],
        "labels": {
            "start_token": [answer["start_token"]],
            "end_token": [answer["end_token"]],
            "category": answer["category"],
        },
    }

  _lowerCAmelCase						:	Optional[Any]        =					input_ids[:q_len]
  _lowerCAmelCase						:	List[Any]        =					range(_lowerCamelCase      ,len(_lowerCamelCase     )      ,max_length - doc_stride     )

  _lowerCAmelCase						:	Tuple        =					[]
  _lowerCAmelCase						:	str        =					[]
  _lowerCAmelCase						:	List[str]        =					[]
  _lowerCAmelCase						:	Any        =					[]  # null, yes, no, long, short
  for i in doc_start_indices:
    _lowerCAmelCase						:	Union[str, Any]        =					i + max_length - q_len
    _lowerCAmelCase						:	int        =					input_ids[i:end_index]
    inputs.append(q_indices + slice     )
    assert len(inputs[-1]     ) <= max_length, "Issue in truncating length"

    if start_token >= i and end_token <= end_index - 1:
      _lowerCAmelCase						:	Dict        =					start_token - i + q_len
      _lowerCAmelCase						:	List[str]        =					end_token - i + q_len
      answers_category.append(answer["""category"""][0]     )  # ["short"] -> "short"
    else:
      _lowerCAmelCase						:	Any        =					-100
      _lowerCAmelCase						:	Tuple        =					-100
      answers_category.append("""null"""     )
    _lowerCAmelCase						:	Tuple        =					inputs[-1][start_token : end_token + 1]

    answers_start_token.append(_lowerCamelCase     )
    answers_end_token.append(_lowerCamelCase     )
    if assertion:
      if new != old and new != [tokenizer.cls_token_id]:
        print("""ISSUE in strided for ID:"""      ,example["""id"""]     )
        print("""New:"""      ,tokenizer.decode(_lowerCamelCase     )     )
        print("""Old:"""      ,tokenizer.decode(_lowerCamelCase     )      ,end="""\n\n"""     )
    if slice[-1] == tokenizer.sep_token_id:
      break

  return {
      "example_id": example["id"],
      "input_ids": inputs,
      "labels": {
          "start_token": answers_start_token,
          "end_token": answers_end_token,
          "category": answers_category,
      },
  }





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	List[Any]      ,_lowerCamelCase						:	Optional[Any]      ,_lowerCamelCase						:	Optional[int]=2048      ,_lowerCamelCase						:	Any=4096      ,_lowerCamelCase						:	List[str]=False     )	->				Optional[Any]:
  _lowerCAmelCase						:	Tuple        =					get_strided_contexts_and_ans(
      _lowerCamelCase      ,_lowerCamelCase      ,doc_stride=_lowerCamelCase      ,max_length=_lowerCamelCase      ,assertion=_lowerCamelCase      ,)

  return example





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	str      ,_lowerCamelCase						:	Optional[int]     )	->				int:
  with jsonlines.open(_lowerCamelCase      ,"""a"""     ) as writer:
    for example in tqdm(_lowerCamelCase      ,total=len(_lowerCamelCase     )      ,desc="""Saving samples ... """     ):
      _lowerCAmelCase						:	Optional[Any]        =					example["""labels"""]
      for ids, start, end, cat in zip(
          example["""input_ids"""]      ,labels["""start_token"""]      ,labels["""end_token"""]      ,labels["""category"""]      ,):
        if start == -1 and end == -1:
          continue  # leave waste samples with no answer
        if cat == "null" and np.random.rand() < 0.6:
          continue  # removing 50 % samples
        writer.write(
            {
                """input_ids""": ids,
                """start_token""": start,
                """end_token""": end,
                """category""": CATEGORY_MAPPING[cat],
            }     )


if __name__ == "__main__":
     from datasets import load_dataset

     from transformers import BigBirdTokenizer

     _a				:		Optional[Any]	   =       load_dataset('natural_questions')
     _a				:		Union[str, Any]	   =       BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')

     _a				:		int	   =       data['train' if PROCESS_TRAIN == 'true' else 'validation']

     _a				:		str	   =       {
         'tokenizer': tokenizer,
         'doc_stride': DOC_STRIDE,
         'max_length': MAX_LENGTH,
         'assertion': False,
     }
     _a				:		Any	   =       data.map(prepare_inputs, fn_kwargs=fn_kwargs)
     _a				:		Optional[int]	   =       data.remove_columns(['annotations', 'document', 'id', 'question'])
     print(data)

     np.random.seed(SEED)
     _a				:		int	   =       'nq-training.jsonl' if PROCESS_TRAIN == 'true' else 'nq-validation.jsonl'
     save_to_disk(data, file_name=cache_file_name)





"""simple docstring"""




from .constants import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    TORCH_LAUNCH_PARAMS,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
)
from .dataclasses import (
    BnbQuantizationConfig,
    ComputeEnvironment,
    CustomDtype,
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    DynamoBackend,
    FPaRecipeKwargs,
    FullyShardedDataParallelPlugin,
    GradientAccumulationPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    KwargsHandler,
    LoggerType,
    MegatronLMPlugin,
    PrecisionType,
    ProjectConfiguration,
    RNGType,
    SageMakerDistributedType,
    TensorInformation,
    TorchDynamoPlugin,
)
from .environment import get_int_from_env, parse_choice_from_env, parse_flag_from_env
from .imports import (
    get_ccl_version,
    is_abit_bnb_available,
    is_abit_bnb_available,
    is_aim_available,
    is_bfaa_available,
    is_bnb_available,
    is_botoa_available,
    is_ccl_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_fpa_available,
    is_ipex_available,
    is_megatron_lm_available,
    is_mlflow_available,
    is_mps_available,
    is_npu_available,
    is_rich_available,
    is_safetensors_available,
    is_sagemaker_available,
    is_tensorboard_available,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)
from .modeling import (
    check_device_map,
    check_tied_parameters_in_config,
    check_tied_parameters_on_same_device,
    compute_module_sizes,
    convert_file_size_to_int,
    dtype_byte_size,
    find_tied_parameters,
    get_balanced_memory,
    get_max_layer_size,
    get_max_memory,
    get_mixed_precision_context_manager,
    id_tensor_storage,
    infer_auto_device_map,
    load_checkpoint_in_model,
    load_offloaded_weights,
    load_state_dict,
    named_module_tensors,
    retie_parameters,
    set_module_tensor_to_device,
    shard_checkpoint,
)
from .offload import (
    OffloadedWeightsLoader,
    PrefixedDataset,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
    save_offload_index,
)
from .operations import (
    broadcast,
    broadcast_object_list,
    concatenate,
    convert_outputs_to_fpaa,
    convert_to_fpaa,
    find_batch_size,
    find_device,
    gather,
    gather_object,
    get_data_structure,
    honor_type,
    initialize_tensors,
    is_namedtuple,
    is_tensor_information,
    is_torch_tensor,
    listify,
    pad_across_processes,
    recursively_apply,
    reduce,
    send_to_device,
    slice_tensors,
)
from .versions import compare_versions, is_torch_version


if is_deepspeed_available():
     from .deepspeed import (
         DeepSpeedEngineWrapper,
         DeepSpeedOptimizerWrapper,
         DeepSpeedSchedulerWrapper,
         DummyOptim,
         DummyScheduler,
         HfDeepSpeedConfig,
     )

from .bnb import has_abit_bnb_layers, load_and_quantize_model
from .fsdp_utils import load_fsdp_model, load_fsdp_optimizer, save_fsdp_model, save_fsdp_optimizer
from .launch import (
    PrepareForLaunch,
    _filter_args,
    prepare_deepspeed_cmd_env,
    prepare_multi_gpu_env,
    prepare_sagemager_args_inputs,
    prepare_simple_launcher_cmd_env,
    prepare_tpu,
)
from .megatron_lm import (
    AbstractTrainStep,
    BertTrainStep,
    GPTTrainStep,
    MegatronEngine,
    MegatronLMDummyDataLoader,
    MegatronLMDummyScheduler,
    MegatronLMOptimizerWrapper,
    MegatronLMSchedulerWrapper,
    TaTrainStep,
    avg_losses_across_data_parallel_group,
    gather_across_data_parallel_groups,
)
from .megatron_lm import initialize as megatron_lm_initialize
from .megatron_lm import prepare_data_loader as megatron_lm_prepare_data_loader
from .megatron_lm import prepare_model as megatron_lm_prepare_model
from .megatron_lm import prepare_optimizer as megatron_lm_prepare_optimizer
from .megatron_lm import prepare_scheduler as megatron_lm_prepare_scheduler
from .memory import find_executable_batch_size, release_memory
from .other import (
    extract_model_from_parallel,
    get_pretty_name,
    is_port_in_use,
    merge_dicts,
    patch_environment,
    save,
    wait_for_everyone,
    write_basic_config,
)
from .random import set_seed, synchronize_rng_state, synchronize_rng_states
from .torch_xla import install_xla
from .tqdm import tqdm
from .transformer_engine import convert_model, has_transformer_engine_layers




