import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


lowerCAmelCase__     						=			re.compile(r'\s+')






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       return {"hash": hashlib.mda(re.sub(lowerCAmelCase__     ,  ''     ,  example['content']     ).encode('utf-8'     )     ).hexdigest()}






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       lowerCAmelCase__					     = [len(lowerCAmelCase__     ) for line in example['content'].splitlines()]
       return {"line_mean": np.mean(lowerCAmelCase__     ), "line_max": max(lowerCAmelCase__     )}






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       lowerCAmelCase__					     = np.mean([c.isalnum() for c in example['content']]     )
       return {"alpha_frac": alpha_frac}






def 				__lowerCamelCase		(						lowerCAmelCase__     ,  lowerCAmelCase__     ):
       if example["hash"] in uniques:
              uniques.remove(example['hash']     )
              return True
       else:
              return False






def 				__lowerCamelCase		(						lowerCAmelCase__     ,  lowerCAmelCase__=5     ):
       lowerCAmelCase__					     = ['auto-generated', 'autogenerated', 'automatically generated']
       lowerCAmelCase__					     = example['content'].splitlines()
       for _, line in zip(range(lowerCAmelCase__     )     ,  lowerCAmelCase__     ):
              for keyword in keywords:
                     if keyword in line.lower():
                            return {"autogenerated": True}
       else:
              return {"autogenerated": False}






def 				__lowerCamelCase		(						lowerCAmelCase__     ,  lowerCAmelCase__=5     ,  lowerCAmelCase__=0.0_5     ):
       lowerCAmelCase__					     = ['unit tests', 'test file', 'configuration file']
       lowerCAmelCase__					     = example['content'].splitlines()
       lowerCAmelCase__					     = 0
       lowerCAmelCase__					     = 0
       # first test
       for _, line in zip(range(lowerCAmelCase__     )     ,  lowerCAmelCase__     ):
              for keyword in keywords:
                     if keyword in line.lower():
                            return {"config_or_test": True}
    # second test
       lowerCAmelCase__					     = example['content'].count('\n'     )
       lowerCAmelCase__					     = int(coeff * nlines     )
       for line in lines:
              count_config += line.lower().count('config'     )
              count_test += line.lower().count('test'     )
              if count_config > threshold or count_test > threshold:
                     return {"config_or_test": True}
       return {"config_or_test": False}






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       lowerCAmelCase__					     = ['def ', 'class ', 'for ', 'while ']
       lowerCAmelCase__					     = example['content'].splitlines()
       for line in lines:
              for keyword in keywords:
                     if keyword in line.lower():
                            return {"has_no_keywords": False}
       return {"has_no_keywords": True}






def 				__lowerCamelCase		(						lowerCAmelCase__     ,  lowerCAmelCase__=4     ):
       lowerCAmelCase__					     = example['content'].splitlines()
       lowerCAmelCase__					     = 0
       for line in lines:
              counter += line.lower().count('='     )
              if counter > minimum:
                     return {"has_few_assignments": False}
       return {"has_few_assignments": True}






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       lowerCAmelCase__					     = tokenizer(example['content']     ,  truncation=lowerCAmelCase__     )['input_ids']
       lowerCAmelCase__					     = len(example['content']     ) / len(lowerCAmelCase__     )
       return {"ratio": ratio}






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       lowerCAmelCase__					     = {}
       results.update(get_hash(lowerCAmelCase__     )     )
       results.update(line_stats(lowerCAmelCase__     )     )
       results.update(alpha_stats(lowerCAmelCase__     )     )
       results.update(char_token_ratio(lowerCAmelCase__     )     )
       results.update(is_autogenerated(lowerCAmelCase__     )     )
       results.update(is_config_or_test(lowerCAmelCase__     )     )
       results.update(has_no_keywords(lowerCAmelCase__     )     )
       results.update(has_few_assignments(lowerCAmelCase__     )     )
       return results






def 				__lowerCamelCase		(						lowerCAmelCase__     ,  lowerCAmelCase__     ,  lowerCAmelCase__     ):
       if not check_uniques(lowerCAmelCase__     ,  lowerCAmelCase__     ):
              return False
       elif example["autogenerated"]:
              return False
       elif example["line_max"] > args.line_max:
              return False
       elif example["line_mean"] > args.line_mean:
              return False
       elif example["alpha_frac"] < args.alpha_frac:
              return False
       elif example["ratio"] < args.min_token_ratio:
              return False
       elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
              return False
       elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
              return False
       elif example["has_few_assignments"]:
              return False
       else:
              return True






def 				__lowerCamelCase		(						lowerCAmelCase__     ):
       with open(lowerCAmelCase__     ,  'rb'     ) as f_in:
              with gzip.open(str(lowerCAmelCase__     ) + '.gz'     ,  'wb'     ,  compresslevel=6     ) as f_out:
                     shutil.copyfileobj(lowerCAmelCase__     ,  lowerCAmelCase__     )
       os.unlink(lowerCAmelCase__     )


# Settings
lowerCAmelCase__     						=			HfArgumentParser(PreprocessingArguments)
lowerCAmelCase__     						=			parser.parse_args()
if args.num_workers is None:
    lowerCAmelCase__     						=			multiprocessing.cpu_count()
lowerCAmelCase__     						=			AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
lowerCAmelCase__     						=			time.time()
lowerCAmelCase__     						=			load_dataset(args.dataset_name, split='train')
print(F'Time to load dataset: {time.time()-t_start:.2f}')

# Run preprocessing
lowerCAmelCase__     						=			time.time()
lowerCAmelCase__     						=			ds.map(preprocess, num_proc=args.num_workers)
print(F'Time to preprocess dataset: {time.time()-t_start:.2f}')

# Deduplicate hashes
lowerCAmelCase__     						=			set(ds.unique('hash'))
lowerCAmelCase__     						=			len(uniques) / len(ds)
print(F'Fraction of duplicates: {1-frac:.2%}')

# Deduplicate data and apply heuristics
lowerCAmelCase__     						=			time.time()
lowerCAmelCase__     						=			ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(F'Time to filter dataset: {time.time()-t_start:.2f}')
print(F'Size of filtered dataset: {len(ds_filter)}')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
    lowerCAmelCase__     						=			time.time()
    lowerCAmelCase__						,		lowerCAmelCase__     						=			deduplicate_dataset(ds_filter, args.jaccard_threshold)
    print(F'Time to deduplicate dataset: {time.time()-t_start:.2f}')
    print(F'Size of deduplicate dataset: {len(ds_filter)}')

# Save data in batches of samples_per_file
lowerCAmelCase__     						=			Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
    with open(output_dir / 'duplicate_clusters.json', 'w') as f:
        json.dump(duplicate_clusters, f)

lowerCAmelCase__     						=			output_dir / 'data'
data_dir.mkdir(exist_ok=True)

lowerCAmelCase__     						=			time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
    lowerCAmelCase__     						=			str(data_dir / F'file-{file_number+1:012}.json')
    lowerCAmelCase__     						=			min(len(ds_filter), index + args.samples_per_file)
    ds_filter.select(list(range(index, end_index))).to_json(file_path)
    compress_file(file_path)



print(F'Time to save dataset: {time.time()-t_start:.2f}')

import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


lowerCAmelCase__     						=			logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm
    import torch_xla.debug.metrics as met
class     a_			(						SCREAMING_SNAKE_CASE						):






 '''simple docstring'''






 def __init__(      self :  int      ,   *lowercase__ :  Dict      ,   lowercase__ :  List[Any]=None      ,   lowercase__ :  Optional[Any]=None      ,   lowercase__ :  Dict=None      ,   **lowercase__ :  Optional[int]):







        '''simple docstring'''



        super().__init__(*lowercase__      ,   **lowercase__)
        lowerCAmelCase__					     = eval_examples
        lowerCAmelCase__					     = post_process_function
        lowerCAmelCase__					     = quant_trainer_args
        lowerCAmelCase__					     = 128  # default number of calibration samples




 def 		__snake_case						(      self :  Tuple      ,   lowercase__ :  Any=None):







        '''simple docstring'''



        if calib_dataset is None and self.calib_dataset is None:
               raise ValueError('Trainer: calibration requires an calib_dataset.')
        lowerCAmelCase__					     = calib_dataset if calib_dataset is not None else self.calib_dataset

        lowerCAmelCase__					     = self._remove_unused_columns(lowercase__      ,   description='Calibration')

        return DataLoader(
            lowercase__      ,   batch_size=self.args.eval_batch_size      ,   collate_fn=self.data_collator      ,   drop_last=self.args.dataloader_drop_last      ,   num_workers=self.args.dataloader_num_workers      ,   pin_memory=self.args.dataloader_pin_memory      ,   shuffle=lowercase__      ,   )




 def 		__snake_case						(      self :  List[Any]      ,   lowercase__ :  Union[str, Any]=None):







        '''simple docstring'''



        lowerCAmelCase__					     = self.train_dataset if calib_dataset is None else calib_dataset
        lowerCAmelCase__					     = self.get_calib_dataloader(lowercase__)

        lowerCAmelCase__					     = self.model
        quant_trainer.configure_model(lowercase__      ,   self.quant_trainer_args      ,   calib=lowercase__)
        model.eval()
        quant_trainer.enable_calibration(lowercase__)

        logger.info('***** Running calibration *****')
        logger.info(F"""  Num examples = {self.calib_num}""")
        logger.info(F"""  Batch size = {calib_dataloader.batch_size}""")

        for step, inputs in enumerate(lowercase__):
               # Prediction step
               lowerCAmelCase__     , lowerCAmelCase__     , lowerCAmelCase__					     = self.prediction_step(lowercase__      ,   lowercase__      ,   prediction_loss_only=lowercase__)
               if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                      break

        quant_trainer.finish_calibration(lowercase__      ,   self.quant_trainer_args)
        lowerCAmelCase__					     = model




 def 		__snake_case						(      self :  Optional[Any]      ,   lowercase__ :  List[Any]=None      ,   lowercase__ :  Optional[Any]=None      ,   lowercase__ :  List[Any]=None      ,   lowercase__ :  str = "eval"):







        '''simple docstring'''



        lowerCAmelCase__					     = self.eval_dataset if eval_dataset is None else eval_dataset
        lowerCAmelCase__					     = self.get_eval_dataloader(lowercase__)
        lowerCAmelCase__					     = self.eval_examples if eval_examples is None else eval_examples

        # Temporarily disable metric computation, we will do it in the loop here.
        lowerCAmelCase__					     = self.compute_metrics
        lowerCAmelCase__					     = None
        lowerCAmelCase__					     = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
               lowerCAmelCase__					     = eval_loop(
                   lowercase__      ,   description='Evaluation'      ,   prediction_loss_only=True if compute_metrics is None else None      ,   ignore_keys=lowercase__      ,   )
        finally:
               lowerCAmelCase__					     = compute_metrics

        if self.post_process_function is not None and self.compute_metrics is not None:
               lowerCAmelCase__					     = self.post_process_function(lowercase__      ,   lowercase__      ,   output.predictions)
               lowerCAmelCase__					     = self.compute_metrics(lowercase__)

               # Prefix all keys with metric_key_prefix + '_'
               for key in list(metrics.keys()):
                      if not key.startswith(F"""{metric_key_prefix}_"""):
                             lowerCAmelCase__					     = metrics.pop(lowercase__)

               self.log(lowercase__)
        else:
               lowerCAmelCase__					     = {}

        if self.args.tpu_metrics_debug or self.args.debug:
               # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
               xm.master_print(met.metrics_report())

        lowerCAmelCase__					     = self.callback_handler.on_evaluate(self.args      ,   self.state      ,   self.control      ,   lowercase__)
        return metrics




 def 		__snake_case						(      self :  Optional[int]      ,   lowercase__ :  str      ,   lowercase__ :  Any      ,   lowercase__ :  List[str]=None      ,   lowercase__ :  str = "test"):







        '''simple docstring'''



        lowerCAmelCase__					     = self.get_test_dataloader(lowercase__)

        # Temporarily disable metric computation, we will do it in the loop here.
        lowerCAmelCase__					     = self.compute_metrics
        lowerCAmelCase__					     = None
        lowerCAmelCase__					     = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
               lowerCAmelCase__					     = eval_loop(
                   lowercase__      ,   description='Prediction'      ,   prediction_loss_only=True if compute_metrics is None else None      ,   ignore_keys=lowercase__      ,   )
        finally:
               lowerCAmelCase__					     = compute_metrics

        if self.post_process_function is None or self.compute_metrics is None:
               return output

        lowerCAmelCase__					     = self.post_process_function(lowercase__      ,   lowercase__      ,   output.predictions      ,   'predict')
        lowerCAmelCase__					     = self.compute_metrics(lowercase__)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()):
               if not key.startswith(F"""{metric_key_prefix}_"""):
                      lowerCAmelCase__					     = metrics.pop(lowercase__)

        return PredictionOutput(predictions=predictions.predictions      ,   label_ids=predictions.label_ids      ,   metrics=lowercase__)




 def 		__snake_case						(      self :  List[str]      ,   lowercase__ :  List[str]="./"):







        '''simple docstring'''



        lowerCAmelCase__					     = self.eval_dataset
        lowerCAmelCase__					     = self.get_eval_dataloader(lowercase__)

        lowerCAmelCase__					     = next(iter(lowercase__))

        # saving device - to make it consistent
        lowerCAmelCase__					     = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

        # convert to tuple
        lowerCAmelCase__					     = tuple(v.to(lowercase__) for k, v in batch.items())

        logger.info('Converting model to be onnx compatible')
        from pytorch_quantization.nn import TensorQuantizer

        lowerCAmelCase__					     = True

        lowerCAmelCase__					     = self.model.to(lowercase__)

        model.eval()
        model.float()

        lowerCAmelCase__					     = model.module if hasattr(lowercase__      ,   'module') else model
        quant_trainer.configure_model(lowercase__      ,   self.quant_trainer_args)

        lowerCAmelCase__					     = os.path.join(lowercase__      ,   'model.onnx')
        logger.info(F"""exporting model to {output_model_file}""")

        lowerCAmelCase__					     = {0: 'batch_size', 1: 'seq_len'}

        torch.onnx.export(
            lowercase__      ,   lowercase__      ,   lowercase__      ,   export_params=lowercase__      ,   opset_version=13      ,   do_constant_folding=lowercase__      ,   input_names=['input_ids', 'attention_mask', 'token_type_ids']      ,   output_names=['output_start_logits', 'output_end_logits']      ,   dynamic_axes={
                'input_ids': axes,
                'attention_mask': axes,
                'token_type_ids': axes,
                'output_start_logits': axes,
                'output_end_logits': axes,
            }      ,   verbose=lowercase__      ,   )
        logger.info('onnx export finished')
