



import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


__snake_case						:int						=					re.compile(r'''\s+''')





def 					__snake_case     (      _UpperCAmelCase    ):
    return {"hash": hashlib.mda(re.sub(_UpperCAmelCase       ,			''''''       ,			example['''content''']    ).encode('''utf-8'''    )    ).hexdigest()}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    [len(_UpperCAmelCase    ) for line in example['''content'''].splitlines()]
    return {"line_mean": np.mean(_UpperCAmelCase    ), "line_max": max(_UpperCAmelCase    )}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    np.mean([c.isalnum() for c in example['''content''']]    )
    return {"alpha_frac": alpha_frac}





def 					__snake_case     (      _UpperCAmelCase       ,			_UpperCAmelCase    ):
    if example["hash"] in uniques:
        uniques.remove(example['''hash''']    )
        return True
    else:
        return False





def 					__snake_case     (      _UpperCAmelCase       ,			_UpperCAmelCase=5    ):
    __a							=    ['''auto-generated''', '''autogenerated''', '''automatically generated''']
    __a							=    example['''content'''].splitlines()
    for _, line in zip(range(_UpperCAmelCase    )       ,			_UpperCAmelCase    ):
        for keyword in keywords:
            if keyword in line.lower():
                return {"autogenerated": True}
    else:
        return {"autogenerated": False}





def 					__snake_case     (      _UpperCAmelCase       ,			_UpperCAmelCase=5       ,			_UpperCAmelCase=0.05    ):
    __a							=    ['''unit tests''', '''test file''', '''configuration file''']
    __a							=    example['''content'''].splitlines()
    __a							=    0
    __a							=    0
    # first test
    for _, line in zip(range(_UpperCAmelCase    )       ,			_UpperCAmelCase    ):
        for keyword in keywords:
            if keyword in line.lower():
                return {"config_or_test": True}
    # second test
    __a							=    example['''content'''].count('''\n'''    )
    __a							=    int(coeff * nlines    )
    for line in lines:
        count_config += line.lower().count('''config'''    )
        count_test += line.lower().count('''test'''    )
        if count_config > threshold or count_test > threshold:
            return {"config_or_test": True}
    return {"config_or_test": False}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    ['''def ''', '''class ''', '''for ''', '''while ''']
    __a							=    example['''content'''].splitlines()
    for line in lines:
        for keyword in keywords:
            if keyword in line.lower():
                return {"has_no_keywords": False}
    return {"has_no_keywords": True}





def 					__snake_case     (      _UpperCAmelCase       ,			_UpperCAmelCase=4    ):
    __a							=    example['''content'''].splitlines()
    __a							=    0
    for line in lines:
        counter += line.lower().count('''='''    )
        if counter > minimum:
            return {"has_few_assignments": False}
    return {"has_few_assignments": True}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    tokenizer(example['''content''']       ,			truncation=_UpperCAmelCase    )['''input_ids''']
    __a							=    len(example['''content''']    ) / len(_UpperCAmelCase    )
    return {"ratio": ratio}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    {}
    results.update(get_hash(_UpperCAmelCase    )    )
    results.update(line_stats(_UpperCAmelCase    )    )
    results.update(alpha_stats(_UpperCAmelCase    )    )
    results.update(char_token_ratio(_UpperCAmelCase    )    )
    results.update(is_autogenerated(_UpperCAmelCase    )    )
    results.update(is_config_or_test(_UpperCAmelCase    )    )
    results.update(has_no_keywords(_UpperCAmelCase    )    )
    results.update(has_few_assignments(_UpperCAmelCase    )    )
    return results





def 					__snake_case     (      _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase    ):
    if not check_uniques(_UpperCAmelCase       ,			_UpperCAmelCase    ):
        return False
    elif example["autogenerated"]:
        return False
    elif example["line_max"] > args.line_max:
        return False
    elif example["line_mean"] > args.line_mean:
        return False
    elif example["alpha_frac"] < args.alpha_frac:
        return False
    elif example["ratio"] < args.min_token_ratio:
        return False
    elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
        return False
    elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
        return False
    elif example["has_few_assignments"]:
        return False
    else:
        return True





def 					__snake_case     (      _UpperCAmelCase    ):
    with open(_UpperCAmelCase       ,			'''rb'''    ) as f_in:
        with gzip.open(str(_UpperCAmelCase    ) + '''.gz'''       ,			'''wb'''       ,			compresslevel=6    ) as f_out:
            shutil.copyfileobj(_UpperCAmelCase       ,			_UpperCAmelCase    )
    os.unlink(_UpperCAmelCase    )


# Settings
__snake_case						:Optional[Any]						=					HfArgumentParser(PreprocessingArguments)
__snake_case						:Tuple						=					parser.parse_args()
if args.num_workers is None:
 __snake_case						:str						=					multiprocessing.cpu_count()
__snake_case						:Optional[int]						=					AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
__snake_case						:Any						=					time.time()
__snake_case						:Tuple						=					load_dataset(args.dataset_name, split='''train''')
print(f'Time to load dataset: {time.time()-t_start:.2f}')

# Run preprocessing
__snake_case						:Optional[Any]						=					time.time()
__snake_case						:Union[str, Any]						=					ds.map(preprocess, num_proc=args.num_workers)
print(f'Time to preprocess dataset: {time.time()-t_start:.2f}')

# Deduplicate hashes
__snake_case						:Union[str, Any]						=					set(ds.unique('''hash'''))
__snake_case						:Any						=					len(uniques) / len(ds)
print(f'Fraction of duplicates: {1-frac:.2%}')

# Deduplicate data and apply heuristics
__snake_case						:List[Any]						=					time.time()
__snake_case						:Optional[int]						=					ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(f'Time to filter dataset: {time.time()-t_start:.2f}')
print(f'Size of filtered dataset: {len(ds_filter)}')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
 __snake_case						:Tuple						=					time.time()
 __snake_case		,__snake_case						:int						=					deduplicate_dataset(ds_filter, args.jaccard_threshold)
 print(f'Time to deduplicate dataset: {time.time()-t_start:.2f}')
 print(f'Size of deduplicate dataset: {len(ds_filter)}')

# Save data in batches of samples_per_file
__snake_case						:List[str]						=					Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
 with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
  json.dump(duplicate_clusters, f)

__snake_case						:Dict						=					output_dir / '''data'''
data_dir.mkdir(exist_ok=True)

__snake_case						:Optional[Any]						=					time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
 __snake_case						:int						=					str(data_dir / f'file-{file_number+1:012}.json')
 __snake_case						:int						=					min(len(ds_filter), index + args.samples_per_file)
 ds_filter.select(list(range(index, end_index))).to_json(file_path)
 compress_file(file_path)
print(f'Time to save dataset: {time.time()-t_start:.2f}')










import os
import warnings
from typing import List, Optional

from ...tokenization_utils_base import BatchEncoding
from ...utils import logging
from .configuration_rag import RagConfig


__snake_case						:int						=					logging.get_logger(__name__)

class    _A	:



       def __init__(		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					Any					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           __a							=    question_encoder
           __a							=    generator
           __a							=    self.question_encoder



       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           if os.path.isfile(__SCREAMING_SNAKE_CASE):
               raise ValueError(F'Provided path ({save_directory}) should be a directory, not a file')
           os.makedirs(__SCREAMING_SNAKE_CASE					,		exist_ok=__SCREAMING_SNAKE_CASE)
           __a							=    os.path.join(__SCREAMING_SNAKE_CASE					,		'''question_encoder_tokenizer''')
           __a							=    os.path.join(__SCREAMING_SNAKE_CASE					,		'''generator_tokenizer''')
           self.question_encoder.save_pretrained(__SCREAMING_SNAKE_CASE)
           self.generator.save_pretrained(__SCREAMING_SNAKE_CASE)



       @classmethod
       def 		_lowerCamelCase      (		cls							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Any					,		**__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           from ..auto.tokenization_auto import AutoTokenizer

           __a							=    kwargs.pop('''config'''					,		__SCREAMING_SNAKE_CASE)

           if config is None:
               __a							=    RagConfig.from_pretrained(__SCREAMING_SNAKE_CASE)

           __a							=    AutoTokenizer.from_pretrained(
               __SCREAMING_SNAKE_CASE					,		config=config.question_encoder					,		subfolder='''question_encoder_tokenizer''')
           __a							=    AutoTokenizer.from_pretrained(
               __SCREAMING_SNAKE_CASE					,		config=config.generator					,		subfolder='''generator_tokenizer''')
           return cls(question_encoder=__SCREAMING_SNAKE_CASE					,		generator=__SCREAMING_SNAKE_CASE)



       def __call__(		self							:					Dict					,		*__SCREAMING_SNAKE_CASE							:					List[Any]					,		**__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           return self.current_tokenizer(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		*__SCREAMING_SNAKE_CASE							:					Dict					,		**__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           return self.generator.batch_decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		*__SCREAMING_SNAKE_CASE							:					List[str]					,		**__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           return self.generator.decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           __a							=    self.question_encoder



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    self.generator





       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[str]] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[int] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[int] = None					,		__SCREAMING_SNAKE_CASE							:					str = "longest"					,		__SCREAMING_SNAKE_CASE							:					str = None					,		__SCREAMING_SNAKE_CASE							:					bool = True					,		**__SCREAMING_SNAKE_CASE							:					List[str]					,		):


           '''simple docstring'''

           warnings.warn(
               '''`prepare_seq2seq_batch` is deprecated and will be removed in version 5 of 🤗 Transformers. Use the '''
               '''regular `__call__` method to prepare your inputs and the tokenizer under the `with_target_tokenizer` '''
               '''context manager to prepare your targets. See the documentation of your specific tokenizer for more '''
               '''details'''					,		__SCREAMING_SNAKE_CASE					,		)
           if max_length is None:
               __a							=    self.current_tokenizer.model_max_length
           __a							=    self(
               __SCREAMING_SNAKE_CASE					,		add_special_tokens=__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		max_length=__SCREAMING_SNAKE_CASE					,		padding=__SCREAMING_SNAKE_CASE					,		truncation=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)
           if tgt_texts is None:
               return model_inputs
           # Process tgt_texts
           if max_target_length is None:
               __a							=    self.current_tokenizer.model_max_length
           __a							=    self(
               text_target=__SCREAMING_SNAKE_CASE					,		add_special_tokens=__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		padding=__SCREAMING_SNAKE_CASE					,		max_length=__SCREAMING_SNAKE_CASE					,		truncation=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)
           __a							=    labels['''input_ids''']
           return model_inputs





