

'''simple docstring'''




import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_lowerCAmelCase         =						re.compile(r"\s+")




def 						UpperCamelCase	(			a			)				->     List[Any]:






     '''simple docstring'''
     return {"hash": hashlib.mda(re.sub(a		,						''''''		,						example['''content''']			).encode('''utf-8'''			)			).hexdigest()}




def 						UpperCamelCase	(			a			)				->     Optional[int]:






     '''simple docstring'''
     __magic_name__						       =    [len(a			) for line in example['''content'''].splitlines()]
     return {"line_mean": np.mean(a			), "line_max": max(a			)}




def 						UpperCamelCase	(			a			)				->     Any:






     '''simple docstring'''
     __magic_name__						       =    np.mean([c.isalnum() for c in example['''content''']]			)
     return {"alpha_frac": alpha_frac}




def 						UpperCamelCase	(			a		,						a			)				->     Any:






     '''simple docstring'''
     if example["hash"] in uniques:
          uniques.remove(example['''hash''']			)
          return True
     else:
          return False




def 						UpperCamelCase	(			a		,						a=5			)				->     List[Any]:






     '''simple docstring'''
     __magic_name__						       =    ['''auto-generated''', '''autogenerated''', '''automatically generated''']
     __magic_name__						       =    example['''content'''].splitlines()
     for _, line in zip(range(a			)		,						a			):
          for keyword in keywords:
               if keyword in line.lower():
                    return {"autogenerated": True}
     else:
          return {"autogenerated": False}




def 						UpperCamelCase	(			a		,						a=5		,						a=0.05			)				->     Any:






     '''simple docstring'''
     __magic_name__						       =    ['''unit tests''', '''test file''', '''configuration file''']
     __magic_name__						       =    example['''content'''].splitlines()
     __magic_name__						       =    0
     __magic_name__						       =    0
     # first test
     for _, line in zip(range(a			)		,						a			):
          for keyword in keywords:
               if keyword in line.lower():
                    return {"config_or_test": True}
    # second test
     __magic_name__						       =    example['''content'''].count('''\n'''			)
     __magic_name__						       =    int(coeff * nlines			)
     for line in lines:
          count_config += line.lower().count('''config'''			)
          count_test += line.lower().count('''test'''			)
          if count_config > threshold or count_test > threshold:
               return {"config_or_test": True}
     return {"config_or_test": False}




def 						UpperCamelCase	(			a			)				->     int:






     '''simple docstring'''
     __magic_name__						       =    ['''def ''', '''class ''', '''for ''', '''while ''']
     __magic_name__						       =    example['''content'''].splitlines()
     for line in lines:
          for keyword in keywords:
               if keyword in line.lower():
                    return {"has_no_keywords": False}
     return {"has_no_keywords": True}




def 						UpperCamelCase	(			a		,						a=4			)				->     List[str]:






     '''simple docstring'''
     __magic_name__						       =    example['''content'''].splitlines()
     __magic_name__						       =    0
     for line in lines:
          counter += line.lower().count('''='''			)
          if counter > minimum:
               return {"has_few_assignments": False}
     return {"has_few_assignments": True}




def 						UpperCamelCase	(			a			)				->     Dict:






     '''simple docstring'''
     __magic_name__						       =    tokenizer(example['''content''']		,						truncation=a			)['''input_ids''']
     __magic_name__						       =    len(example['''content''']			) / len(a			)
     return {"ratio": ratio}




def 						UpperCamelCase	(			a			)				->     int:






     '''simple docstring'''
     __magic_name__						       =    {}
     results.update(get_hash(a			)			)
     results.update(line_stats(a			)			)
     results.update(alpha_stats(a			)			)
     results.update(char_token_ratio(a			)			)
     results.update(is_autogenerated(a			)			)
     results.update(is_config_or_test(a			)			)
     results.update(has_no_keywords(a			)			)
     results.update(has_few_assignments(a			)			)
     return results




def 						UpperCamelCase	(			a		,						a		,						a			)				->     str:






     '''simple docstring'''
     if not check_uniques(a		,						a			):
          return False
     elif example["autogenerated"]:
          return False
     elif example["line_max"] > args.line_max:
          return False
     elif example["line_mean"] > args.line_mean:
          return False
     elif example["alpha_frac"] < args.alpha_frac:
          return False
     elif example["ratio"] < args.min_token_ratio:
          return False
     elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
          return False
     elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
          return False
     elif example["has_few_assignments"]:
          return False
     else:
          return True




def 						UpperCamelCase	(			a			)				->     Optional[Any]:






     '''simple docstring'''
     with open(a		,						'''rb'''			) as f_in:
          with gzip.open(str(a			) + '''.gz'''		,						'''wb'''		,						compresslevel=6			) as f_out:
               shutil.copyfileobj(a		,						a			)
     os.unlink(a			)


# Settings
_lowerCAmelCase         =						HfArgumentParser(PreprocessingArguments)
_lowerCAmelCase         =						parser.parse_args()
if args.num_workers is None:
      _lowerCAmelCase         =						multiprocessing.cpu_count()
_lowerCAmelCase         =						AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_lowerCAmelCase         =						time.time()
_lowerCAmelCase         =						load_dataset(args.dataset_name, split="train")
print(F'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
_lowerCAmelCase         =						time.time()
_lowerCAmelCase         =						ds.map(preprocess, num_proc=args.num_workers)
print(F'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
_lowerCAmelCase         =						set(ds.unique("hash"))
_lowerCAmelCase         =						len(uniques) / len(ds)
print(F'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
_lowerCAmelCase         =						time.time()
_lowerCAmelCase         =						ds.filter(filter, fn_kwargs={"uniques": uniques, "args": args})
print(F'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(F'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
      _lowerCAmelCase         =						time.time()
      _lowerCAmelCase      ,				_lowerCAmelCase         =						deduplicate_dataset(ds_filter, args.jaccard_threshold)
      print(F'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
      print(F'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
_lowerCAmelCase         =						Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
      with open(output_dir / "duplicate_clusters.json", "w") as f:
            json.dump(duplicate_clusters, f)

_lowerCAmelCase         =						output_dir / "data"
data_dir.mkdir(exist_ok=True)

_lowerCAmelCase         =						time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
      _lowerCAmelCase         =						str(data_dir / F'''file-{file_number+1:012}.json''')
      _lowerCAmelCase         =						min(len(ds_filter), index + args.samples_per_file)
      ds_filter.select(list(range(index, end_index))).to_json(file_path)
      compress_file(file_path)


print(F'''Time to save dataset: {time.time()-t_start:.2f}''')









'''simple docstring'''




from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
      from ...feature_extraction_utils import FeatureExtractionMixin
      from ...tokenization_utils_base import PreTrainedTokenizerBase
      from ...utils import TensorType

_lowerCAmelCase         =						logging.get_logger(__name__)

_lowerCAmelCase         =						{
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]






class   _SCREAMING_SNAKE_CASE   (   __a     ):
 __SCREAMING_SNAKE_CASE					:str      	=					"""whisper"""
 __SCREAMING_SNAKE_CASE					:str      	=					["""past_key_values"""]
 __SCREAMING_SNAKE_CASE					:Tuple      	=					{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



 def __init__(   self		:   Dict			,				a__		:   Optional[int]=5_1865			,				a__		:   str=80			,				a__		:   List[str]=6			,				a__		:   List[str]=4			,				a__		:   List[Any]=6			,				a__		:   Union[str, Any]=4			,				a__		:   Tuple=1536			,				a__		:   Optional[int]=1536			,				a__		:   List[str]=0.0			,				a__		:   Union[str, Any]=0.0			,				a__		:   Union[str, Any]=5_0257			,				a__		:   Dict=True			,				a__		:   Optional[Any]=True			,				a__		:   Union[str, Any]="gelu"			,				a__		:   Tuple=256			,				a__		:   Dict=0.0			,				a__		:   str=0.0			,				a__		:   Optional[Any]=0.0			,				a__		:   int=0.02			,				a__		:   Any=False			,				a__		:   List[Any]=1500			,				a__		:   Optional[int]=448			,				a__		:   Dict=5_0256			,				a__		:   str=5_0256			,				a__		:   Tuple=5_0256			,				a__		:   List[str]=None			,				a__		:   List[Any]=[220, 5_0256]			,				a__		:   Any=False			,				a__		:   Dict=256			,				a__		:   Optional[Any]=False			,				a__		:   str=0.05			,				a__		:   List[Any]=10			,				a__		:   List[Any]=2			,				a__		:   Optional[int]=0.0			,				a__		:   List[Any]=10			,				a__		:   Union[str, Any]=0			,				a__		:   int=7			,				**a__		:   Any			,				):
      __magic_name__						       =    vocab_size
      __magic_name__						       =    num_mel_bins
      __magic_name__						       =    d_model
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    encoder_attention_heads
      __magic_name__						       =    decoder_layers
      __magic_name__						       =    decoder_attention_heads
      __magic_name__						       =    decoder_ffn_dim
      __magic_name__						       =    encoder_ffn_dim
      __magic_name__						       =    dropout
      __magic_name__						       =    attention_dropout
      __magic_name__						       =    activation_dropout
      __magic_name__						       =    activation_function
      __magic_name__						       =    init_std
      __magic_name__						       =    encoder_layerdrop
      __magic_name__						       =    decoder_layerdrop
      __magic_name__						       =    use_cache
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    scale_embedding  # scale factor will be sqrt(d_model) if True
      __magic_name__						       =    max_source_positions
      __magic_name__						       =    max_target_positions

      # Audio Classification-specific parameters. Feel free to ignore for other classes.
      __magic_name__						       =    classifier_proj_size
      __magic_name__						       =    use_weighted_layer_sum

      # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
      __magic_name__						       =    apply_spec_augment
      __magic_name__						       =    mask_time_prob
      __magic_name__						       =    mask_time_length
      __magic_name__						       =    mask_time_min_masks
      __magic_name__						       =    mask_feature_prob
      __magic_name__						       =    mask_feature_length
      __magic_name__						       =    mask_feature_min_masks

      __magic_name__						       =    median_filter_width

      super().__init__(
          pad_token_id=a__			,				bos_token_id=a__			,				eos_token_id=a__			,				is_encoder_decoder=a__			,				decoder_start_token_id=a__			,				suppress_tokens=a__			,				begin_suppress_tokens=a__			,				**a__			,				)






class   _SCREAMING_SNAKE_CASE   (   __a     ):



 @property
 def 			snake_case__  (   self		:   List[str]      ):
      __magic_name__						       =    OrderedDict(
          [
              ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
          ]      )
      if self.use_past:
           __magic_name__						       =    {0: '''batch'''}
      else:
           __magic_name__						       =    {0: '''batch''', 1: '''decoder_sequence'''}

      if self.use_past:
           self.fill_with_past_key_values_(a__			,				direction='''inputs'''      )

      return common_inputs



 def 			snake_case__  (   self		:   Optional[int]			,				a__		:   Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]			,				a__		:   int = -1			,				a__		:   int = -1			,				a__		:   bool = False			,				a__		:   Optional["TensorType"] = None			,				a__		:   int = 2_2050			,				a__		:   float = 5.0			,				a__		:   int = 220			,				):
      __magic_name__						       =    OrderedDict()
      __magic_name__						       =    OnnxConfig.generate_dummy_inputs(
          self			,				preprocessor=preprocessor.feature_extractor			,				batch_size=a__			,				framework=a__			,				sampling_rate=a__			,				time_duration=a__			,				frequency=a__			,				)
      __magic_name__						       =    encoder_inputs['''input_features'''].shape[2]
      __magic_name__						       =    encoder_sequence_length // 2 if self.use_past else seq_length

      __magic_name__						       =    super().generate_dummy_inputs(
          preprocessor.tokenizer			,				a__			,				a__			,				a__			,				a__      )

      __magic_name__						       =    encoder_inputs.pop('''input_features'''      )
      __magic_name__						       =    decoder_inputs.pop('''decoder_input_ids'''      )

      if "past_key_values" in decoder_inputs:
           __magic_name__						       =    decoder_inputs.pop('''past_key_values'''      )

      return dummy_inputs



 @property
 def 			snake_case__  (   self		:   Dict      ):
      return 1E-3






