


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case							:	Any					      =				logging.getLogger(__name__)






def 							__lowercase       (			):
							a__					=					argparse.ArgumentParser(
							    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'       )
							parser.add_argument(
							    '--dataset_name' ,   type=__lowerCAmelCase ,   default='wikitext' ,   help='Name of the training. Explore datasets at: hf.co/datasets.' ,   )
							parser.add_argument(
							    '--dataset_config' ,   type=__lowerCAmelCase ,   default='wikitext-103-raw-v1' ,   help='Configuration name of the dataset.'       )
							parser.add_argument(
							    '--tokenizer_name_or_path' ,   type=__lowerCAmelCase ,   default='sayakpaul/unigram-tokenizer-wikitext' ,   help='Tokenizer identifier. Can be a local filepath or a Hub identifier.' ,   )
							parser.add_argument(
							    '--shard_size' ,   type=__lowerCAmelCase ,   default=1_0_0_0 ,   help='Number of entries to go in a single shard.' ,   )
							parser.add_argument('--split' ,   type=__lowerCAmelCase ,   default='train' ,   choices=['train', 'test', 'validation']       )
							parser.add_argument(
							    '--limit' ,   default=__lowerCAmelCase ,   type=__lowerCAmelCase ,   help='Limit the number of shards (used for debugging).' ,   )
							parser.add_argument(
							    '--max_length' ,   type=__lowerCAmelCase ,   default=5_1_2 ,   help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
							    ' sequence length that is a multiple of 8.' ,   )
							parser.add_argument(
							    '--output_dir' ,   default='tf-tpu' ,   type=__lowerCAmelCase ,   help='Output directory where the TFRecord shards will be saved. If the'
							    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
							    ' shards will be directly saved to a Google Cloud Storage bucket.' ,   )

							a__					=					parser.parse_args()
							return args






def 							__lowercase       (			__lowerCAmelCase							:    Any       ):
							def fn(__lowerCAmelCase							:    Any       ):
														return tokenizer(examples['text']       )

							return fn






def 							__lowercase       (			__lowerCAmelCase							:    Union[str, Any]       ):
							a__					=					[]
							for i in range(len(tokenized_data['input_ids']       )       ):
														a__					=					{
														    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]       )       ),
														    'attention_mask': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]       )       ),
														}
														a__					=					tf.train.Features(feature=__lowerCAmelCase       )
														a__					=					tf.train.Example(features=__lowerCAmelCase       )
														a__					=					example.SerializeToString()
														records.append(__lowerCAmelCase       )
							return records






def 							__lowercase       (			__lowerCAmelCase							:    List[str]       ):
							a__					=					datasets.load_dataset(args.dataset_name ,   args.dataset_config ,   split=args.split       )

							if args.limit is not None:
														a__					=					min(len(__lowerCAmelCase       ) ,   args.limit       )
														a__					=					dataset.select(range(__lowerCAmelCase       )       )
														print(F'Limiting the dataset to {args.limit} entries.'       )

							a__					=					AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir       ):
																					os.makedirs(args.output_dir       )
														a__					=					os.path.join(args.output_dir ,   args.split       )
														if not os.path.exists(__lowerCAmelCase       ):
																					os.makedirs(__lowerCAmelCase       )
							else:
														a__					=					os.path.join(args.output_dir ,   args.split       )

							# Tokenize the whole dataset at once.
							a__					=					tokenize_function(__lowerCAmelCase       )
							a__					=					dataset.map(__lowerCAmelCase ,   batched=__lowerCAmelCase ,   num_proc=4 ,   remove_columns=['text']       )

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(__lowerCAmelCase							:    int       ):
														# Concatenate all texts.
														a__					=					{k: sum(examples[k] ,   []       ) for k in examples.keys()}
														a__					=					len(concatenated_examples[list(examples.keys()       )[0]]       )
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														a__					=					(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														a__					=					{
														    k: [t[i : i + args.max_length] for i in range(0 ,   __lowerCAmelCase ,   args.max_length       )]
														    for k, t in concatenated_examples.items()
														}
														return result

							a__					=					dataset_tokenized.map(__lowerCAmelCase ,   batched=__lowerCAmelCase ,   batch_size=1_0_0_0 ,   num_proc=4       )

							a__					=					0
							a__					=					0
							for shard in range(0 ,   len(__lowerCAmelCase       ) ,   args.shard_size       ):
														a__					=					grouped_dataset[shard : shard + args.shard_size]
														a__					=					len(dataset_snapshot['input_ids']       )
														a__					=					os.path.join(__lowerCAmelCase ,   F'dataset-{shard_count}-{records_containing}.tfrecord'       )
														a__					=					get_serialized_examples(__lowerCAmelCase       )

														with tf.io.TFRecordWriter(__lowerCAmelCase       ) as out_file:
																					for i in range(len(__lowerCAmelCase       )       ):
																												a__					=					serialized_examples[i]
																												out_file.write(__lowerCAmelCase       )
																					print('Wrote file {} containing {} records'.format(__lowerCAmelCase ,   __lowerCAmelCase       )       )

														shard_count += 1
														total_records += records_containing

							with open(F'split-{args.split}-records-count.txt' ,   'w'       ) as f:
														print(F'Total {args.split} records: {total_records}' ,   file=__lowerCAmelCase       )


if __name__ == "__main__":
						snake_case							:	str					      =				parse_args()
						main(args)








import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command







class 	snake_case_       (unittest.TestCase				):
					UpperCAmelCase__ :   Optional[Any]   				=      inspect.getfile(accelerate.test_utils				)
					UpperCAmelCase__ :   Tuple   				=      os.path.sep.join(mod_file.split(os.path.sep				)[:-1] + ['''scripts''', '''test_cli.py''']				)

					UpperCAmelCase__ :   Optional[int]   				=      ['''accelerate''', '''launch''']
					UpperCAmelCase__ :   List[str]   				=      Path.home() / '''.cache/huggingface/accelerate'''
					UpperCAmelCase__ :   Union[str, Any]   				=      '''default_config.yaml'''
					UpperCAmelCase__ :   List[Any]   				=      config_folder / config_file
					UpperCAmelCase__ :   Optional[int]   				=      config_folder / '''_default_config.yaml'''

					UpperCAmelCase__ :   int   				=      Path('''tests/test_configs'''				)
					@classmethod
					def 			lowerCamelCase__(				cls       :Any				) ->   int:
												if cls.config_path.is_file():
																			cls.config_path.rename(cls.changed_path				)
					@classmethod
					def 			lowerCamelCase__(				cls       :Dict				) ->   Union[str, Any]:
												if cls.changed_path.is_file():
																			cls.changed_path.rename(cls.config_path				)
					def 			lowerCamelCase__(				self       :Any				) ->   Optional[int]:
												a__					=					self.base_cmd
												if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
																			cmd += ["--multi_gpu"]
												execute_subprocess_async(cmd + [self.test_file_path]      ,env=os.environ.copy()				)
					def 			lowerCamelCase__(				self       :str				) ->   List[Any]:
												for config in sorted(self.test_config_path.glob('**/*.yaml'				)				):
																			with self.subTest(config_file=__snake_case				):
																										execute_subprocess_async(
																										    self.base_cmd + ['--config_file', str(__snake_case				), self.test_file_path]      ,env=os.environ.copy()				)



					def 			lowerCamelCase__(				self       :int				) ->   List[str]:
												execute_subprocess_async(['accelerate', 'test']      ,env=os.environ.copy()				)







class 	snake_case_       (unittest.TestCase				):
					UpperCAmelCase__ :   List[Any]   				=      '''test-tpu'''
					UpperCAmelCase__ :   str   				=      '''us-central1-a'''
					UpperCAmelCase__ :   Optional[Any]   				=      '''ls'''
					UpperCAmelCase__ :   Optional[int]   				=      ['''accelerate''', '''tpu-config''']
					UpperCAmelCase__ :   Any   				=      '''cd /usr/share'''
					UpperCAmelCase__ :   Tuple   				=      '''tests/test_samples/test_command_file.sh'''
					UpperCAmelCase__ :   List[Any]   				=      '''Running gcloud compute tpus tpu-vm ssh'''
					def 			lowerCamelCase__(				self       :Optional[Any]				) ->   Tuple:
												a__					=					run_command(
												    self.cmd
												    + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug']      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :int				) ->   str:
												a__					=					run_command(
												    self.cmd
												    + [
												        '--config_file',
												        'tests/test_configs/0_12_0.yaml',
												        '--command',
												        self.command,
												        '--tpu_zone',
												        self.tpu_zone,
												        '--tpu_name',
												        self.tpu_name,
												        '--debug',
												    ]      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :Any				) ->   Dict:
												a__					=					run_command(
												    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug']      ,return_stdout=__snake_case				)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :Tuple				) ->   str:
												a__					=					run_command(
												    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug']      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :Dict				) ->   int:
												a__					=					run_command(
												    self.cmd
												    + [
												        '--config_file',
												        'tests/test_configs/latest.yaml',
												        '--command',
												        self.command,
												        '--command',
												        'echo "Hello World"',
												        '--debug',
												    ]      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :Union[str, Any]				) ->   List[Any]:
												a__					=					run_command(
												    self.cmd
												    + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug']      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :int				) ->   int:
												a__					=					run_command(
												    self.cmd
												    + [
												        '--config_file',
												        'tests/test_configs/0_12_0.yaml',
												        '--command_file',
												        self.command_file,
												        '--tpu_zone',
												        self.tpu_zone,
												        '--tpu_name',
												        self.tpu_name,
												        '--debug',
												    ]      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'      ,__snake_case      ,)
					def 			lowerCamelCase__(				self       :Union[str, Any]				) ->   List[Any]:
												a__					=					run_command(
												    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug']      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all'      ,__snake_case      ,)



					def 			lowerCamelCase__(				self       :Any				) ->   Optional[int]:
												a__					=					run_command(
												    self.cmd
												    + [
												        '--config_file',
												        'tests/test_configs/latest.yaml',
												        '--install_accelerate',
												        '--accelerate_version',
												        '12.0.0',
												        '--debug',
												    ]      ,return_stdout=__snake_case      ,)
												self.assertIn(
												    F'{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all'      ,__snake_case      ,)




