

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__snake_case      :      Optional[int]             =       logging.getLogger(__name__)







def 		_UpperCamelCase				(      )					->				Optional[Any]:

						"""simple docstring"""



						lowerCAmelCase__        =  argparse.ArgumentParser(
						    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'						)
						parser.add_argument(
						    '--dataset_name'      ,    type=UpperCamelCase_      ,    default='wikitext'      ,    help='Name of the training. Explore datasets at: hf.co/datasets.'      ,    )
						parser.add_argument(
						    '--dataset_config'      ,    type=UpperCamelCase_      ,    default='wikitext-103-raw-v1'      ,    help='Configuration name of the dataset.'						)
						parser.add_argument(
						    '--tokenizer_name_or_path'      ,    type=UpperCamelCase_      ,    default='sayakpaul/unigram-tokenizer-wikitext'      ,    help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'      ,    )
						parser.add_argument(
						    '--shard_size'      ,    type=UpperCamelCase_      ,    default=1000      ,    help='Number of entries to go in a single shard.'      ,    )
						parser.add_argument('--split'      ,    type=UpperCamelCase_      ,    default='train'      ,    choices=['train', 'test', 'validation']						)
						parser.add_argument(
						    '--limit'      ,    default=UpperCamelCase_      ,    type=UpperCamelCase_      ,    help='Limit the number of shards (used for debugging).'      ,    )
						parser.add_argument(
						    '--max_length'      ,    type=UpperCamelCase_      ,    default=512      ,    help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
						    ' sequence length that is a multiple of 8.'      ,    )
						parser.add_argument(
						    '--output_dir'      ,    default='tf-tpu'      ,    type=UpperCamelCase_      ,    help='Output directory where the TFRecord shards will be saved. If the'
						    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
						    ' shards will be directly saved to a Google Cloud Storage bucket.'      ,    )

						lowerCAmelCase__        =  parser.parse_args()
						return args







def 		_UpperCamelCase				(      UpperCamelCase_    :   int						)					->				List[Any]:

						"""simple docstring"""



						def fn(UpperCamelCase_    :   Optional[Any]						):
												return tokenizer(examples['text']						)

						return fn







def 		_UpperCamelCase				(      UpperCamelCase_    :   Tuple						)					->				List[str]:

						"""simple docstring"""



						lowerCAmelCase__        =  []
						for i in range(len(tokenized_data['input_ids']						)						):
												lowerCAmelCase__        =  {
												    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]						)						),
												    'attention_mask': tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]						)						),
												}
												lowerCAmelCase__        =  tf.train.Features(feature=UpperCamelCase_						)
												lowerCAmelCase__        =  tf.train.Example(features=UpperCamelCase_						)
												lowerCAmelCase__        =  example.SerializeToString()
												records.append(UpperCamelCase_						)
						return records







def 		_UpperCamelCase				(      UpperCamelCase_    :   Any						)					->				Tuple:

						"""simple docstring"""



						lowerCAmelCase__        =  datasets.load_dataset(args.dataset_name      ,    args.dataset_config      ,    split=args.split						)

						if args.limit is not None:
												lowerCAmelCase__        =  min(len(UpperCamelCase_						)      ,    args.limit						)
												lowerCAmelCase__        =  dataset.select(range(UpperCamelCase_						)						)
												print(F"Limiting the dataset to {args.limit} entries."						)

						lowerCAmelCase__        =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir						):
																		os.makedirs(args.output_dir						)
												lowerCAmelCase__        =  os.path.join(args.output_dir      ,    args.split						)
												if not os.path.exists(UpperCamelCase_						):
																		os.makedirs(UpperCamelCase_						)
						else:
												lowerCAmelCase__        =  os.path.join(args.output_dir      ,    args.split						)

						# Tokenize the whole dataset at once.
						lowerCAmelCase__        =  tokenize_function(UpperCamelCase_						)
						lowerCAmelCase__        =  dataset.map(UpperCamelCase_      ,    batched=UpperCamelCase_      ,    num_proc=4      ,    remove_columns=['text']						)

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(UpperCamelCase_    :   Tuple						):
												# Concatenate all texts.
												lowerCAmelCase__        =  {k: sum(examples[k]      ,    []						) for k in examples.keys()}
												lowerCAmelCase__        =  len(concatenated_examples[list(examples.keys()						)[0]]						)
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												lowerCAmelCase__        =  (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												lowerCAmelCase__        =  {
												    k: [t[i : i + args.max_length] for i in range(0      ,    UpperCamelCase_      ,    args.max_length						)]
												    for k, t in concatenated_examples.items()
												}
												return result

						lowerCAmelCase__        =  dataset_tokenized.map(UpperCamelCase_      ,    batched=UpperCamelCase_      ,    batch_size=1000      ,    num_proc=4						)

						lowerCAmelCase__        =  0
						lowerCAmelCase__        =  0
						for shard in range(0      ,    len(UpperCamelCase_						)      ,    args.shard_size						):
												lowerCAmelCase__        =  grouped_dataset[shard : shard + args.shard_size]
												lowerCAmelCase__        =  len(dataset_snapshot['input_ids']						)
												lowerCAmelCase__        =  os.path.join(UpperCamelCase_      ,    F"dataset-{shard_count}-{records_containing}.tfrecord"						)
												lowerCAmelCase__        =  get_serialized_examples(UpperCamelCase_						)

												with tf.io.TFRecordWriter(UpperCamelCase_						) as out_file:
																		for i in range(len(UpperCamelCase_						)						):
																								lowerCAmelCase__        =  serialized_examples[i]
																								out_file.write(UpperCamelCase_						)
																		print('Wrote file {} containing {} records'.format(UpperCamelCase_      ,    UpperCamelCase_						)						)

												shard_count += 1
												total_records += records_containing

						with open(F"split-{args.split}-records-count.txt"      ,    'w'						) as f:
												print(F"Total {args.split} records: {total_records}"      ,    file=UpperCamelCase_						)


if __name__ == "__main__":
					__snake_case      :      Any             =       parse_args()
					main(args)













import qiskit







def 		_UpperCamelCase				(      UpperCamelCase_    :   int      ,    UpperCamelCase_    :   int						)					->				qiskit.result.counts.Counts:

						"""simple docstring"""



						lowerCAmelCase__        =  qiskit.Aer.get_backend('aer_simulator'						)

						# Create a Quantum Circuit acting on the q register
						lowerCAmelCase__        =  qiskit.QuantumCircuit(UpperCamelCase_      ,    UpperCamelCase_						)

						# Map the quantum measurement to the classical bits
						circuit.measure([0]      ,    [0]						)

						# Execute the circuit on the simulator
						lowerCAmelCase__        =  qiskit.execute(UpperCamelCase_      ,    UpperCamelCase_      ,    shots=1000						)

						# Return the histogram data of the results of the experiment.
						return job.result().get_counts(UpperCamelCase_						)


if __name__ == "__main__":
					print(f'Total count for various states are: {single_qubit_measure(1, 1)}')





