



import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


__A		:					Tuple							=re.compile(r'''\s+''')






def      __lowercase       (			a__       )    ->			Dict:
							return {"hash": hashlib.mda(re.sub(lowercase_							,       ''							,       example['content']       ).encode('utf-8'       )       ).hexdigest()}






def      __lowercase       (			a__       )    ->			Dict:
							__SCREAMING_SNAKE_CASE									= [len(lowercase_       ) for line in example['content'].splitlines()]
							return {"line_mean": np.mean(lowercase_       ), "line_max": max(lowercase_       )}






def      __lowercase       (			a__       )    ->			List[Any]:
							__SCREAMING_SNAKE_CASE									= np.mean([c.isalnum() for c in example['content']]       )
							return {"alpha_frac": alpha_frac}






def      __lowercase       (			a__							,       a__       )    ->			int:





							if example["hash"] in uniques:
														uniques.remove(example['hash']       )
														return True
							else:
														return False






def      __lowercase       (			a__							,       a__=5       )    ->			int:
							__SCREAMING_SNAKE_CASE									= ['auto-generated', 'autogenerated', 'automatically generated']
							__SCREAMING_SNAKE_CASE									= example['content'].splitlines()
							for _, line in zip(range(lowercase_       )							,       lowercase_       ):
														for keyword in keywords:
																					if keyword in line.lower():
																												return {"autogenerated": True}
							else:
														return {"autogenerated": False}






def      __lowercase       (			a__							,       a__=5							,       a__=0.05       )    ->			List[str]:
							__SCREAMING_SNAKE_CASE									= ['unit tests', 'test file', 'configuration file']
							__SCREAMING_SNAKE_CASE									= example['content'].splitlines()
							__SCREAMING_SNAKE_CASE									= 0
							__SCREAMING_SNAKE_CASE									= 0
							# first test
							for _, line in zip(range(lowercase_       )							,       lowercase_       ):
														for keyword in keywords:
																					if keyword in line.lower():
																												return {"config_or_test": True}
														# second test
							__SCREAMING_SNAKE_CASE									= example['content'].count('\n'       )
							__SCREAMING_SNAKE_CASE									= int(coeff * nlines       )
							for line in lines:
														count_config += line.lower().count('config'       )
														count_test += line.lower().count('test'       )
														if count_config > threshold or count_test > threshold:
																					return {"config_or_test": True}
							return {"config_or_test": False}






def      __lowercase       (			a__       )    ->			List[str]:
							__SCREAMING_SNAKE_CASE									= ['def ', 'class ', 'for ', 'while ']
							__SCREAMING_SNAKE_CASE									= example['content'].splitlines()
							for line in lines:
														for keyword in keywords:
																					if keyword in line.lower():
																												return {"has_no_keywords": False}
							return {"has_no_keywords": True}






def      __lowercase       (			a__							,       a__=4       )    ->			int:
							__SCREAMING_SNAKE_CASE									= example['content'].splitlines()
							__SCREAMING_SNAKE_CASE									= 0
							for line in lines:
														counter += line.lower().count('='       )
														if counter > minimum:
																					return {"has_few_assignments": False}
							return {"has_few_assignments": True}






def      __lowercase       (			a__       )    ->			Any:
							__SCREAMING_SNAKE_CASE									= tokenizer(example['content']							,       truncation=lowercase_       )['input_ids']
							__SCREAMING_SNAKE_CASE									= len(example['content']       ) / len(lowercase_       )
							return {"ratio": ratio}






def      __lowercase       (			a__       )    ->			List[Any]:
							__SCREAMING_SNAKE_CASE									= {}
							results.update(get_hash(lowercase_       )       )
							results.update(line_stats(lowercase_       )       )
							results.update(alpha_stats(lowercase_       )       )
							results.update(char_token_ratio(lowercase_       )       )
							results.update(is_autogenerated(lowercase_       )       )
							results.update(is_config_or_test(lowercase_       )       )
							results.update(has_no_keywords(lowercase_       )       )
							results.update(has_few_assignments(lowercase_       )       )
							return results






def      __lowercase       (			a__							,       a__							,       a__       )    ->			Tuple:





							if not check_uniques(lowercase_							,       lowercase_       ):
														return False
							elif example["autogenerated"]:
														return False
							elif example["line_max"] > args.line_max:
														return False
							elif example["line_mean"] > args.line_mean:
														return False
							elif example["alpha_frac"] < args.alpha_frac:
														return False
							elif example["ratio"] < args.min_token_ratio:
														return False
							elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
														return False
							elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
														return False
							elif example["has_few_assignments"]:
														return False
							else:
														return True






def      __lowercase       (			a__       )    ->			Tuple:





							with open(lowercase_							,       'rb'       ) as f_in:
														with gzip.open(str(lowercase_       ) + '.gz'							,       'wb'							,       compresslevel=6       ) as f_out:
																					shutil.copyfileobj(lowercase_							,       lowercase_       )
							os.unlink(lowercase_       )


# Settings
__A		:					str							=HfArgumentParser(PreprocessingArguments)
__A		:					Any							=parser.parse_args()
if args.num_workers is None:
			__A		:					Union[str, Any]							=multiprocessing.cpu_count()
__A		:					Optional[int]							=AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
__A		:					Any							=time.time()
__A		:					int							=load_dataset(args.dataset_name, split='''train''')
print(F'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
__A		:					Optional[int]							=time.time()
__A		:					Optional[int]							=ds.map(preprocess, num_proc=args.num_workers)
print(F'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
__A		:					Union[str, Any]							=set(ds.unique('''hash'''))
__A		:					Tuple							=len(uniques) / len(ds)
print(F'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
__A		:					List[Any]							=time.time()
__A		:					Tuple							=ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(F'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(F'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
			__A		:					Dict							=time.time()
			__A						,					__A		:					Union[str, Any]							=deduplicate_dataset(ds_filter, args.jaccard_threshold)
			print(F'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
			print(F'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
__A		:					Optional[Any]							=Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
			with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
						json.dump(duplicate_clusters, f)

__A		:					Any							=output_dir / '''data'''
data_dir.mkdir(exist_ok=True)

__A		:					Optional[int]							=time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
			__A		:					Tuple							=str(data_dir / F'''file-{file_number+1:012}.json''')
			__A		:					Union[str, Any]							=min(len(ds_filter), index + args.samples_per_file)
			ds_filter.select(list(range(index, end_index))).to_json(file_path)
			compress_file(file_path)

print(F'''Time to save dataset: {time.time()-t_start:.2f}''')











import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
			from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
			lowerCAmelCase__		:					Optional[int]							=True
			from torch.cuda.amp import autocast


lowerCAmelCase__		:					List[Any]							=logging.getLogger(__name__)

@dataclass
class        UpperCAmelCase_  :

						'''simple docstring'''




						UpperCamelCase__      :			str      =  field(
						    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}	)
						UpperCamelCase__      :			Optional[str]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}   ,   )
						UpperCamelCase__      :			Optional[bool]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}	)
						UpperCamelCase__      :			Optional[bool]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Whether to log verbose messages or not.'''}   ,   )
						UpperCamelCase__      :			Optional[float]      =  field(
						    default=2.0   ,   metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}	)
						UpperCamelCase__      :			Optional[float]      =  field(
						    default=0.5   ,   metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}	)
						UpperCamelCase__      :			Optional[float]      =  field(
						    default=0.9_9_9_9_9_5   ,   metadata={'''help''': '''Decay of gumbel temperature during training.'''}	)







def      __lowercase       (			a__							,       a__       )    ->			Dict:
							logging.basicConfig(
							    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'							,       datefmt='%m/%d/%Y %H:%M:%S'							,       handlers=[logging.StreamHandler(sys.stdout       )]							,       )
							__SCREAMING_SNAKE_CASE									= logging.WARNING
							if model_args.verbose_logging:
														__SCREAMING_SNAKE_CASE									= logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank       ):
														__SCREAMING_SNAKE_CASE									= logging.INFO
							logger.setLevel(a__       )

@dataclass
class        UpperCAmelCase_  :

						'''simple docstring'''




						UpperCamelCase__      :			str      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}	)
						UpperCamelCase__      :			Optional[str]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}	)
						UpperCamelCase__      :			Optional[str]      =  field(
						    default='''train'''   ,   metadata={
						        '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
						    }   ,   )
						UpperCamelCase__      :			Optional[str]      =  field(
						    default='''validation'''   ,   metadata={
						        '''help''': (
						            '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
						        )
						    }   ,   )
						UpperCamelCase__      :			Optional[str]      =  field(
						    default='''file'''   ,   metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}   ,   )
						UpperCamelCase__      :			bool      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}	)
						UpperCamelCase__      :			Optional[int]      =  field(
						    default=1   ,   metadata={
						        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
						    }   ,   )
						UpperCamelCase__      :			Optional[int]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''The number of processes to use for the preprocessing.'''}   ,   )
						UpperCamelCase__      :			Optional[float]      =  field(
						    default=2_0.0   ,   metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}	)







@dataclass
class        UpperCAmelCase_  :

						'''simple docstring'''




						UpperCamelCase__      :			WavaVecaForPreTraining
						UpperCamelCase__      :			WavaVecaFeatureExtractor
						UpperCamelCase__      :			Union[bool, str]      =  "longest"
						UpperCamelCase__      :			Optional[int]      =  None
						UpperCamelCase__      :			Optional[int]      =  None






						def __call__(   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.feature_extractor.pad(
													    _A				,							max_length=self.max_length				,							padding=self.padding				,							pad_to_multiple_of=self.pad_to_multiple_of				,							return_tensors='pt'				,							)
													__SCREAMING_SNAKE_CASE									= self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1]		)

													__SCREAMING_SNAKE_CASE									= batch['input_values'].shape[0]

													# make sure that no loss is computed on padded inputs
													if batch["attention_mask"] is not None:
																				# compute real output lengths according to convolution formula
																				__SCREAMING_SNAKE_CASE									= self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1		)		).to(
																				    torch.long		)

																				__SCREAMING_SNAKE_CASE									= torch.zeros(
																				    (batch_size, mask_indices_seq_length)				,							dtype=torch.long				,							device=batch['input_values'].device		)

																				# these two operations makes sure that all values
																				# before the output lengths indices are attended to
																				__SCREAMING_SNAKE_CASE									= 1
																				__SCREAMING_SNAKE_CASE									= attention_mask.flip([-1]		).cumsum(-1		).flip([-1]		).bool()

													# sample randomly masked indices
													__SCREAMING_SNAKE_CASE									= _compute_mask_indices(
													    (batch_size, mask_indices_seq_length)				,							self.model.config.mask_time_prob				,							self.model.config.mask_time_length				,							attention_mask=_A				,							min_masks=2				,							)

													return batch







class        UpperCAmelCase_  (       UpperCamelCase_	):

						'''simple docstring'''




						def __init__(   self				,							*_A				,							_A=1				,							_A=0				,							_A=1.0				,							**_A		):







													'''simple docstring'''

													super().__init__(*_A				,							**_A		)
													__SCREAMING_SNAKE_CASE									= 0
													__SCREAMING_SNAKE_CASE									= max_gumbel_temp
													__SCREAMING_SNAKE_CASE									= min_gumbel_temp
													__SCREAMING_SNAKE_CASE									= gumbel_temp_decay






						def 	_A     (   self				,							_A				,							_A		):







													'''simple docstring'''

													model.train()
													__SCREAMING_SNAKE_CASE									= self._prepare_inputs(_A		)

													if self.use_amp:
																				with autocast():
																											__SCREAMING_SNAKE_CASE									= self.compute_loss(_A				,							_A		)
													else:
																				__SCREAMING_SNAKE_CASE									= self.compute_loss(_A				,							_A		)

													if self.args.n_gpu > 1 or self.deepspeed:
																				if model.module.config.ctc_loss_reduction == "mean":
																											__SCREAMING_SNAKE_CASE									= loss.mean()
																				elif model.module.config.ctc_loss_reduction == "sum":
																											__SCREAMING_SNAKE_CASE									= loss.sum() / (inputs['mask_time_indices']).sum()
																				else:
																											raise ValueError(f"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""		)

													if self.args.gradient_accumulation_steps > 1:
																				__SCREAMING_SNAKE_CASE									= loss / self.args.gradient_accumulation_steps

													if self.use_amp:
																				self.scaler.scale(_A		).backward()
													elif self.use_apex:
																				with amp.scale_loss(_A				,							self.optimizer		) as scaled_loss:
																											scaled_loss.backward()
													elif self.deepspeed:
																				self.deepspeed.backward(_A		)
													else:
																				loss.backward()

													self.num_update_step += 1
													# make sure gumbel softmax temperature is decayed
													if self.args.n_gpu > 1 or self.deepspeed:
																				model.module.set_gumbel_temperature(
																				    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,							self.min_gumbel_temp		)		)
													else:
																				model.set_gumbel_temperature(
																				    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,							self.min_gumbel_temp		)		)

													return loss.detach()







def      __lowercase       (			)    ->			Tuple:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							__SCREAMING_SNAKE_CASE									= HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

							__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= parser.parse_args_into_dataclasses()
							configure_logger(a__							,       a__       )

							# Downloading and loading a dataset from the hub.
							__SCREAMING_SNAKE_CASE									= load_dataset(data_args.dataset_name							,       data_args.dataset_config_name							,       cache_dir=model_args.cache_dir       )

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														__SCREAMING_SNAKE_CASE									= DatasetDict()
														__SCREAMING_SNAKE_CASE									= load_dataset(
														    data_args.dataset_name							,       data_args.dataset_config_name							,       split=f"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""							,       cache_dir=model_args.cache_dir							,       )
														__SCREAMING_SNAKE_CASE									= load_dataset(
														    data_args.dataset_name							,       data_args.dataset_config_name							,       split=f"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""							,       cache_dir=model_args.cache_dir							,       )
							else:
														# make sure only "validation" and "train" keys remain"
														__SCREAMING_SNAKE_CASE									= DatasetDict()
														__SCREAMING_SNAKE_CASE									= load_dataset(
														    data_args.dataset_name							,       data_args.dataset_config_name							,       split='validation'							,       cache_dir=model_args.cache_dir							,       )
														__SCREAMING_SNAKE_CASE									= load_dataset(
														    data_args.dataset_name							,       data_args.dataset_config_name							,       split=f"""{data_args.train_split_name}"""							,       cache_dir=model_args.cache_dir							,       )

							# only normalized-inputs-training is supported
							__SCREAMING_SNAKE_CASE									= WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path							,       cache_dir=model_args.cache_dir							,       do_normalize=a__       )

							def prepare_dataset(a__       ):
														# check that all files have the correct sampling rate
														__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= librosa.load(batch[data_args.speech_file_column]							,       sr=feature_extractor.sampling_rate       )
														return batch

							# load audio files into numpy arrays
							__SCREAMING_SNAKE_CASE									= datasets.map(
							    a__							,       num_proc=data_args.preprocessing_num_workers							,       remove_columns=datasets['train'].column_names       )

							# filter audio files that are too long
							__SCREAMING_SNAKE_CASE									= vectorized_datasets.filter(
							    lambda a__       : len(data['speech']       ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate       )       )

							def normalize(a__       ):
														return feature_extractor(batch['speech']							,       sampling_rate=feature_extractor.sampling_rate       )

							# normalize and transform to `BatchFeatures`
							__SCREAMING_SNAKE_CASE									= vectorized_datasets.map(
							    a__							,       batched=a__							,       num_proc=data_args.preprocessing_num_workers							,       load_from_cache_file=not data_args.overwrite_cache							,       remove_columns=vectorized_datasets['train'].column_names							,       )

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							__SCREAMING_SNAKE_CASE									= WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path							,       cache_dir=model_args.cache_dir							,       gradient_checkpointing=training_args.gradient_checkpointing							,       )

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
														    ' ``config.feat_extract_norm=\'layer\''       )

							__SCREAMING_SNAKE_CASE									= WavaVecaForPreTraining(a__       )

							__SCREAMING_SNAKE_CASE									= DataCollatorForWavaVecaPretraining(model=a__							,       feature_extractor=a__       )

							__SCREAMING_SNAKE_CASE									= WavaVecaPreTrainer(
							    model=a__							,       data_collator=a__							,       args=a__							,       train_dataset=vectorized_datasets['train']							,       eval_dataset=vectorized_datasets['validation']							,       tokenizer=a__							,       max_gumbel_temp=model_args.max_gumbel_temperature							,       min_gumbel_temp=model_args.min_gumbel_temperature							,       gumbel_temp_decay=model_args.gumbel_temperature_decay							,       )
							trainer.train()


if __name__ == "__main__":
			main()






