



'''simple docstring'''

import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


__lowerCAmelCase						      =       re.compile(r'''\s+''')



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				Any:
		return {"hash": hashlib.mda(re.sub(_UpperCAmelCase						,      ''						,      example['content']  ).encode('utf-8'  )  ).hexdigest()}



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				List[str]:
		_a					:				Optional[int]				=							[len(_UpperCAmelCase  ) for line in example["content"].splitlines()]
		return {"line_mean": np.mean(_UpperCAmelCase  ), "line_max": max(_UpperCAmelCase  )}



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				Any:
		_a					:				int				=							np.mean([c.isalnum() for c in example['content']]  )
		return {"alpha_frac": alpha_frac}



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_  )      ->				Dict:
		if example["hash"] in uniques:
				uniques.remove(example['hash']  )
				return True
		else:
				return False



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_=5  )      ->				Optional[Any]:
		_a					:				Optional[Any]				=							["auto-generated", "autogenerated", "automatically generated"]
		_a					:				Optional[Any]				=							example["content"].splitlines()
		for _, line in zip(range(_UpperCAmelCase  )						,      _UpperCAmelCase  ):
				for keyword in keywords:
						if keyword in line.lower():
								return {"autogenerated": True}
		else:
				return {"autogenerated": False}



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_=5						,      lowerCAmelCase_=0.05  )      ->				int:
		_a					:				Union[str, Any]				=							["unit tests", "test file", "configuration file"]
		_a					:				List[str]				=							example["content"].splitlines()
		_a					:				List[Any]				=							0
		_a					:				Optional[int]				=							0
		# first test
		for _, line in zip(range(_UpperCAmelCase  )						,      _UpperCAmelCase  ):
				for keyword in keywords:
						if keyword in line.lower():
								return {"config_or_test": True}
    # second test
		_a					:				Union[str, Any]				=							example["content"].count('\n'  )
		_a					:				List[Any]				=							int(coeff * nlines  )
		for line in lines:
				count_config += line.lower().count('config'  )
				count_test += line.lower().count('test'  )
				if count_config > threshold or count_test > threshold:
						return {"config_or_test": True}
		return {"config_or_test": False}



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				Optional[Any]:
		_a					:				List[str]				=							["def ", "class ", "for ", "while "]
		_a					:				Tuple				=							example["content"].splitlines()
		for line in lines:
				for keyword in keywords:
						if keyword in line.lower():
								return {"has_no_keywords": False}
		return {"has_no_keywords": True}



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_=4  )      ->				Dict:
		_a					:				Dict				=							example["content"].splitlines()
		_a					:				Union[str, Any]				=							0
		for line in lines:
				counter += line.lower().count('='  )
				if counter > minimum:
						return {"has_few_assignments": False}
		return {"has_few_assignments": True}



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				List[Any]:
		_a					:				Tuple				=							tokenizer(example['content']						,      truncation=_UpperCAmelCase  )["input_ids"]
		_a					:				Dict				=							len(example['content']  ) / len(_UpperCAmelCase  )
		return {"ratio": ratio}



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				Any:
		_a					:				int				=							{}
		results.update(get_hash(_UpperCAmelCase  )  )
		results.update(line_stats(_UpperCAmelCase  )  )
		results.update(alpha_stats(_UpperCAmelCase  )  )
		results.update(char_token_ratio(_UpperCAmelCase  )  )
		results.update(is_autogenerated(_UpperCAmelCase  )  )
		results.update(is_config_or_test(_UpperCAmelCase  )  )
		results.update(has_no_keywords(_UpperCAmelCase  )  )
		results.update(has_few_assignments(_UpperCAmelCase  )  )
		return results



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )      ->				Dict:
		if not check_uniques(_UpperCAmelCase						,      _UpperCAmelCase  ):
				return False
		elif example["autogenerated"]:
				return False
		elif example["line_max"] > args.line_max:
				return False
		elif example["line_mean"] > args.line_mean:
				return False
		elif example["alpha_frac"] < args.alpha_frac:
				return False
		elif example["ratio"] < args.min_token_ratio:
				return False
		elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
				return False
		elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
				return False
		elif example["has_few_assignments"]:
				return False
		else:
				return True



def 					__lowerCamelCase					(				lowerCAmelCase_  )      ->				Union[str, Any]:
		with open(_UpperCAmelCase						,      'rb'  ) as f_in:
				with gzip.open(str(_UpperCAmelCase  ) + '.gz'						,      'wb'						,      compresslevel=6  ) as f_out:
						shutil.copyfileobj(_UpperCAmelCase						,      _UpperCAmelCase  )
		os.unlink(_UpperCAmelCase  )


# Settings
__lowerCAmelCase						      =       HfArgumentParser(PreprocessingArguments)
__lowerCAmelCase						      =       parser.parse_args()
if args.num_workers is None:
							__lowerCAmelCase						      =       multiprocessing.cpu_count()
__lowerCAmelCase						      =       AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
__lowerCAmelCase						      =       time.time()
__lowerCAmelCase						      =       load_dataset(args.dataset_name, split='''train''')
print(f"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
__lowerCAmelCase						      =       time.time()
__lowerCAmelCase						      =       ds.map(preprocess, num_proc=args.num_workers)
print(f"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
__lowerCAmelCase						      =       set(ds.unique('''hash'''))
__lowerCAmelCase						      =       len(uniques) / len(ds)
print(f"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
__lowerCAmelCase						      =       time.time()
__lowerCAmelCase						      =       ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(f"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(f"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
							__lowerCAmelCase						      =       time.time()
							__lowerCAmelCase						      =       deduplicate_dataset(ds_filter, args.jaccard_threshold)
							print(f"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
							print(f"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
__lowerCAmelCase						      =       Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
							with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
														json.dump(duplicate_clusters, f)

__lowerCAmelCase						      =       output_dir / "data"
data_dir.mkdir(exist_ok=True)

__lowerCAmelCase						      =       time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
							__lowerCAmelCase						      =       str(data_dir / f"""file-{file_number+1:012}.json""")
							__lowerCAmelCase						      =       min(len(ds_filter), index + args.samples_per_file)
							ds_filter.select(list(range(index, end_index))).to_json(file_path)
							compress_file(file_path)
print(f"""Time to save dataset: {time.time()-t_start:.2f}""")









'''simple docstring'''

__lowerCAmelCase						      =       range(2, 20 + 1)
__lowerCAmelCase						      =       [10**k for k in range(ks[-1] + 1)]
__lowerCAmelCase						      =       {}



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )      ->				Optional[Any]:
		_a					:				Optional[int]				=							sum(a_i[j] for j in range(lowerCAmelCase_						,      len(lowerCAmelCase_  )  )  )
		_a					:				List[str]				=							sum(a_i[j] * base[j] for j in range(min(len(lowerCAmelCase_  )						,      lowerCAmelCase_  )  )  )

		_a	, _a					:				Any				=							0, 0
		_a					:				Any				=							n - i

		_a					:				List[Any]				=							memo.get(lowerCAmelCase_  )

		if sub_memo is not None:
				_a					:				Tuple				=							sub_memo.get(lowerCAmelCase_  )

				if jumps is not None and len(lowerCAmelCase_  ) > 0:
						# find and make the largest jump without going over
						_a					:				Any				=							-1
						for _k in range(len(lowerCAmelCase_  ) - 1						,      -1						,      -1  ):
								if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
										_a					:				Any				=							_k
										break

						if max_jump >= 0:
								_a	, _a	, _a					:				Tuple				=							jumps[max_jump]
								# since the difference between jumps is cached, add c
								_a					:				Union[str, Any]				=							diff + c
								for j in range(min(lowerCAmelCase_						,      len(lowerCAmelCase_  )  )  ):
										_a	, _a					:				Dict				=							divmod(lowerCAmelCase_						,      10  )
								if new_c > 0:
										add(lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )

				else:
						_a					:				Tuple				=							[]
		else:
				_a					:				Any				=							{c: []}
				_a					:				Optional[Any]				=							sub_memo

		if dn >= max_dn or c + diff >= base[k]:
				return diff, dn

		if k > ks[0]:
				while True:
						# keep doing smaller jumps
						_a	, _a					:				Dict				=							next_term(lowerCAmelCase_						,      k - 1						,      i + dn						,      lowerCAmelCase_  )
						diff += _diff
						dn += terms_jumped

						if dn >= max_dn or c + diff >= base[k]:
								break
		else:
				# would be too small a jump, just compute sequential terms instead
				_a	, _a					:				Any				=							compute(lowerCAmelCase_						,      lowerCAmelCase_						,      i + dn						,      lowerCAmelCase_  )
				diff += _diff
				dn += terms_jumped

		_a					:				Tuple				=							sub_memo[c]

		# keep jumps sorted by # of terms skipped
		_a					:				Any				=							0
		while j < len(lowerCAmelCase_  ):
				if jumps[j][1] > dn:
						break
				j += 1

		# cache the jump for this value digitsum(b) and c
		sub_memo[c].insert(lowerCAmelCase_						,      (diff, dn, k)  )
		return (diff, dn)



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )      ->				str:
		if i >= n:
				return 0, i
		if k > len(lowerCAmelCase_  ):
				a_i.extend([0 for _ in range(k - len(lowerCAmelCase_  )  )]  )

		# note: a_i -> b * 10^k + c
		# ds_b -> digitsum(b)
		# ds_c -> digitsum(c)
		_a					:				Any				=							i
		_a	, _a	, _a					:				Optional[int]				=							0, 0, 0
		for j in range(len(lowerCAmelCase_  )  ):
				if j >= k:
						ds_b += a_i[j]
				else:
						ds_c += a_i[j]

		while i < n:
				i += 1
				_a					:				Any				=							ds_c + ds_b
				diff += addend
				_a					:				int				=							0
				for j in range(lowerCAmelCase_  ):
						_a					:				Optional[Any]				=							a_i[j] + addend
						_a	, _a					:				Tuple				=							divmod(lowerCAmelCase_						,      10  )

						ds_c += a_i[j]

				if addend > 0:
						break

		if addend > 0:
				add(lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )
		return diff, i - start_i



def 					__lowerCamelCase					(				lowerCAmelCase_						,      lowerCAmelCase_						,      lowerCAmelCase_  )      ->				int:
		for j in range(lowerCAmelCase_						,      len(lowerCAmelCase_  )  ):
				_a					:				Optional[Any]				=							digits[j] + addend
				if s >= 10:
						_a	, _a					:				List[str]				=							divmod(lowerCAmelCase_						,      10  )
						_a					:				List[str]				=							addend // 10 + quotient
				else:
						_a					:				Optional[Any]				=							s
						_a					:				Optional[Any]				=							addend // 10

				if addend == 0:
						break

		while addend > 0:
				_a	, _a					:				List[str]				=							divmod(lowerCAmelCase_						,      10  )
				digits.append(lowerCAmelCase_  )



def 					__lowerCamelCase					(				lowerCAmelCase_ = 10**15  )      ->				int:
		_a					:				Dict				=							[1]
		_a					:				int				=							1
		_a					:				Tuple				=							0
		while True:
				_a	, _a					:				str				=							next_term(lowerCAmelCase_						,      20						,      i + dn						,      lowerCAmelCase_  )
				dn += terms_jumped
				if dn == n - i:
						break

		_a					:				Union[str, Any]				=							0
		for j in range(len(lowerCAmelCase_  )  ):
				a_n += digits[j] * 10**j
		return a_n


if __name__ == "__main__":
							print(f"""{solution() = }""")




