"""simple docstring"""
import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



class 					lowerCAmelCase_      (     lowerCAmelCase       ,      unittest.TestCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      List[str]     					=  CanineTokenizer
				_lowerCAmelCase       :      Optional[Any]     					=  False

				def 						snake_case	(			self      ):






										"""simple docstring"""

										super().setUp()
										snake_case        =     CanineTokenizer()
										tokenizer.save_pretrained(self.tmpdirname      )

				@cached_property
				def 						snake_case	(			self      ):






										"""simple docstring"""

										return CanineTokenizer.from_pretrained('google/canine-s'      )

				def 						snake_case	(			self						,      **lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     self.tokenizer_class.from_pretrained(self.tmpdirname						,      **lowerCAmelCase      )
										snake_case        =     10_24
										return tokenizer

				@require_torch
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.canine_tokenizer
										snake_case        =     ['Life is like a box of chocolates.', 'You never know what you\'re gonna get.']
										# fmt: off
										snake_case        =     [5_73_44, 76, 1_05, 1_02, 1_01, 32, 1_05, 1_15, 32, 1_08, 1_05, 1_07, 1_01, 32, 97, 32, 98, 1_11, 1_20, 32, 1_11, 1_02, 32, 99, 1_04, 1_11, 99, 1_11, 1_08, 97, 1_16, 1_01, 1_15, 46, 5_73_45, 0, 0, 0, 0]
										# fmt: on
										snake_case        =     tokenizer(lowerCAmelCase						,      padding=lowerCAmelCase						,      return_tensors='pt'      )
										self.assertIsInstance(lowerCAmelCase						,      lowerCAmelCase      )

										snake_case        =     list(batch.input_ids.numpy()[0]      )

										self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

										self.assertEqual((2, 39)						,      batch.input_ids.shape      )
										self.assertEqual((2, 39)						,      batch.attention_mask.shape      )

				@require_torch
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.canine_tokenizer
										snake_case        =     ['Once there was a man.', 'He wrote a test in HuggingFace Tranformers.']
										snake_case        =     tokenizer(lowerCAmelCase						,      padding=lowerCAmelCase						,      return_tensors='pt'      )
										# check if input_ids, attention_mask and token_type_ids are returned
										self.assertIn('input_ids'						,      lowerCAmelCase      )
										self.assertIn('attention_mask'						,      lowerCAmelCase      )
										self.assertIn('token_type_ids'						,      lowerCAmelCase      )

				@require_torch
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.canine_tokenizer
										snake_case        =     [
										    'What\'s the weater?',
										    'It\'s about 25 degrees.',
										]
										snake_case        =     tokenizer(
										    text_target=lowerCAmelCase						,      max_length=32						,      padding='max_length'						,      truncation=lowerCAmelCase						,      return_tensors='pt'      )
										self.assertEqual(32						,      targets['input_ids'].shape[1]      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers()
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						self.assertNotEqual(tokenizer.model_max_length						,      42      )

        # Now let's start the test
										snake_case        =     self.get_tokenizers()
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						# Isolate this from the other tests because we save additional tokens/etc
																						snake_case        =     tempfile.mkdtemp()

																						snake_case        =     ' He is very happy, UNwant\u00E9d,running'
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						tokenizer.save_pretrained(lowerCAmelCase      )

																						snake_case        =     tokenizer.__class__.from_pretrained(lowerCAmelCase      )
																						snake_case        =     after_tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

																						shutil.rmtree(lowerCAmelCase      )

										snake_case        =     self.get_tokenizers(model_max_length=42      )
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						# Isolate this from the other tests because we save additional tokens/etc
																						snake_case        =     tempfile.mkdtemp()

																						snake_case        =     ' He is very happy, UNwant\u00E9d,running'

																						snake_case        =     tokenizer.additional_special_tokens

																						# We can add a new special token for Canine as follows:
																						snake_case        =     chr(0xE_0_0_7      )
																						additional_special_tokens.append(lowerCAmelCase      )
																						tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}      )
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						tokenizer.save_pretrained(lowerCAmelCase      )

																						snake_case        =     tokenizer.__class__.from_pretrained(lowerCAmelCase      )
																						snake_case        =     after_tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )
																						self.assertIn(lowerCAmelCase						,      after_tokenizer.additional_special_tokens      )
																						self.assertEqual(after_tokenizer.model_max_length						,      42      )

																						snake_case        =     tokenizer.__class__.from_pretrained(lowerCAmelCase						,      model_max_length=43      )
																						self.assertEqual(tokenizer.model_max_length						,      43      )

																						shutil.rmtree(lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers(do_lower_case=lowerCAmelCase      )
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						snake_case   ,snake_case        =     self.get_clean_sequence(lowerCAmelCase      )

																						# a special token for Canine can be defined as follows:
																						snake_case        =     0xE_0_0_5
																						snake_case        =     chr(lowerCAmelCase      )

																						tokenizer.add_special_tokens({'cls_token': special_token}      )
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						self.assertEqual(len(lowerCAmelCase      )						,      1      )

																						snake_case        =     tokenizer.decode(ids + encoded_special_token						,      clean_up_tokenization_spaces=lowerCAmelCase      )
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )

																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						self.assertEqual(lowerCAmelCase						,      input_encoded + special_token_id      )

																						snake_case        =     tokenizer.decode(lowerCAmelCase						,      skip_special_tokens=lowerCAmelCase      )
																						self.assertTrue(special_token not in decoded      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers(do_lower_case=lowerCAmelCase      )
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						snake_case        =     chr(0xE_0_0_5      )
																						snake_case        =     chr(0xE_0_0_6      )

																						# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
																						tokenizer.add_tokens([SPECIAL_TOKEN_1]						,      special_tokens=lowerCAmelCase      )
																						# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
																						# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
																						tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}      )

																						snake_case        =     tokenizer.tokenize(lowerCAmelCase      )
																						snake_case        =     tokenizer.tokenize(lowerCAmelCase      )

																						self.assertEqual(len(lowerCAmelCase      )						,      1      )
																						self.assertEqual(len(lowerCAmelCase      )						,      1      )
																						self.assertEqual(token_a[0]						,      lowerCAmelCase      )
																						self.assertEqual(token_a[0]						,      lowerCAmelCase      )

				@require_tokenizers
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers(do_lower_case=lowerCAmelCase      )
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						# a special token for Canine can be defined as follows:
																						snake_case        =     0xE_0_0_6
																						snake_case        =     chr(lowerCAmelCase      )

																						snake_case        =     AddedToken(lowerCAmelCase						,      lstrip=lowerCAmelCase      )
																						tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}      )

																						with tempfile.TemporaryDirectory() as tmp_dir_name:
																												tokenizer.save_pretrained(lowerCAmelCase      )
																												tokenizer.from_pretrained(lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     []
										if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

										if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer_utils.save_pretrained(lowerCAmelCase      )

																						with open(os.path.join(lowerCAmelCase						,      'special_tokens_map.json'      )						,      encoding='utf-8'      ) as json_file:
																												snake_case        =     json.load(lowerCAmelCase      )

																						with open(os.path.join(lowerCAmelCase						,      'tokenizer_config.json'      )						,      encoding='utf-8'      ) as json_file:
																												snake_case        =     json.load(lowerCAmelCase      )

																						# a special token for Canine can be defined as follows:
																						snake_case        =     0xE_0_0_6
																						snake_case        =     chr(lowerCAmelCase      )

																						snake_case        =     [new_token_a]
																						snake_case        =     [new_token_a]

																						with open(os.path.join(lowerCAmelCase						,      'special_tokens_map.json'      )						,      'w'						,      encoding='utf-8'      ) as outfile:
																												json.dump(lowerCAmelCase						,      lowerCAmelCase      )
																						with open(os.path.join(lowerCAmelCase						,      'tokenizer_config.json'      )						,      'w'						,      encoding='utf-8'      ) as outfile:
																												json.dump(lowerCAmelCase						,      lowerCAmelCase      )

																						# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																						# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																						# "special_tokens_map.json" files
																						snake_case        =     tokenizer_class.from_pretrained(lowerCAmelCase						,      extra_ids=0      )
																						self.assertIn(lowerCAmelCase						,      tokenizer_without_change_in_init.additional_special_tokens      )
																						# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																						self.assertEqual(
																						    [new_token_a]						,      tokenizer_without_change_in_init.convert_ids_to_tokens(
																						        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]      )      )						,      )

																						snake_case        =     0xE_0_0_7
																						snake_case        =     chr(lowerCAmelCase      )
																						# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																						snake_case        =     [AddedToken(lowerCAmelCase						,      lstrip=lowerCAmelCase      )]
																						snake_case        =     tokenizer_class.from_pretrained(
																						    lowerCAmelCase						,      additional_special_tokens=lowerCAmelCase						,      extra_ids=0      )

																						self.assertIn(lowerCAmelCase						,      tokenizer.additional_special_tokens      )
																						# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
																						self.assertEqual(
																						    [new_token_a]						,      tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]      )      )      )

				@require_tokenizers
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers(do_lower_case=lowerCAmelCase      )
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						snake_case        =     'hello world'
																						if self.space_between_special_tokens:
																												snake_case        =     '[CLS] hello world [SEP]'
																						else:
																												snake_case        =     input
																						snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
																						snake_case        =     tokenizer.decode(lowerCAmelCase						,      spaces_between_special_tokens=self.space_between_special_tokens      )
																						self.assertIn(lowerCAmelCase						,      [output, output.lower()]      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.get_tokenizers()
										for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}"""      ):
																						snake_case        =     [
																						    'bos_token',
																						    'eos_token',
																						    'unk_token',
																						    'sep_token',
																						    'pad_token',
																						    'cls_token',
																						    'mask_token',
																						]

										snake_case        =     'a'
										snake_case        =     ord(lowerCAmelCase      )

										for attr in attributes_list:
																setattr(lowerCAmelCase						,      attr + '_id'						,      lowerCAmelCase      )
																self.assertEqual(getattr(lowerCAmelCase						,      lowerCAmelCase      )						,      lowerCAmelCase      )
																self.assertEqual(getattr(lowerCAmelCase						,      attr + '_id'      )						,      lowerCAmelCase      )

																setattr(lowerCAmelCase						,      attr + '_id'						,      lowerCAmelCase      )
																self.assertEqual(getattr(lowerCAmelCase						,      lowerCAmelCase      )						,      lowerCAmelCase      )
																self.assertEqual(getattr(lowerCAmelCase						,      attr + '_id'      )						,      lowerCAmelCase      )

										setattr(lowerCAmelCase						,      'additional_special_tokens_ids'						,      []      )
										self.assertListEqual(getattr(lowerCAmelCase						,      'additional_special_tokens'      )						,      []      )
										self.assertListEqual(getattr(lowerCAmelCase						,      'additional_special_tokens_ids'      )						,      []      )

										snake_case        =     0xE_0_0_6
										snake_case        =     chr(lowerCAmelCase      )
										setattr(lowerCAmelCase						,      'additional_special_tokens_ids'						,      [additional_special_token_id]      )
										self.assertListEqual(getattr(lowerCAmelCase						,      'additional_special_tokens'      )						,      [additional_special_token]      )
										self.assertListEqual(getattr(lowerCAmelCase						,      'additional_special_tokens_ids'      )						,      [additional_special_token_id]      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass



"""simple docstring"""
import asyncio
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from distutils.util import strtobool
from functools import partial
from pathlib import Path
from typing import List, Union
from unittest import mock

import torch

from ..state import AcceleratorState, PartialState
from ..utils import (
    gather,
    is_bnb_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_mps_available,
    is_safetensors_available,
    is_tensorboard_available,
    is_torch_version,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Tuple	,     _UpperCamelCase    :			Optional[int]=False   )   ->			Any:





						"""simple docstring"""






						try:
												snake_case        =     os.environ[key]
						except KeyError:
												# KEY isn't set, default to `default`.
												snake_case        =     default
						else:
												# KEY is set, convert it to True or False.
												try:
																		snake_case        =     strtobool(_UpperCamelCase   )
												except ValueError:
																		# More values are supported, but let's keep the message simple.
																		raise ValueError(f"""If set, {key} must be yes or no."""   )
						return _value


SCREAMING_SNAKE_CASE__   =					parse_flag_from_env("RUN_SLOW", default=False)





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[Any]   )   ->			str:





						"""simple docstring"""






						return unittest.skip('Test was skipped'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[Any]   )   ->			List[str]:





						"""simple docstring"""






						return unittest.skipUnless(_run_slow_tests	,     'test is slow'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[Any]   )   ->			int:





						"""simple docstring"""






						return unittest.skipUnless(not torch.cuda.is_available()	,     'test requires only a CPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Tuple   )   ->			Tuple:





						"""simple docstring"""






						return unittest.skipUnless(torch.cuda.is_available()	,     'test requires a GPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict   )   ->			Optional[int]:





						"""simple docstring"""






						return unittest.skipUnless(is_xpu_available()	,     'test requires a XPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict   )   ->			Any:





						"""simple docstring"""






						return unittest.skipUnless(is_mps_available()	,     'test requires a `mps` backend support in `torch`'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Tuple   )   ->			str:





						"""simple docstring"""






						return unittest.skipUnless(
						    is_transformers_available() and is_datasets_available()	,     'test requires the Hugging Face suite'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[str]   )   ->			Dict:





						"""simple docstring"""






						return unittest.skipUnless(is_bnb_available()	,     'test requires the bitsandbytes library'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[str]   )   ->			List[Any]:





						"""simple docstring"""






						return unittest.skipUnless(is_tpu_available()	,     'test requires TPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict   )   ->			int:





						"""simple docstring"""






						return unittest.skipUnless(torch.cuda.device_count() == 1	,     'test requires a GPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			str   )   ->			List[Any]:





						"""simple docstring"""






						return unittest.skipUnless(torch.xpu.device_count() == 1	,     'test requires a XPU'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Optional[Any]   )   ->			List[Any]:





						"""simple docstring"""






						return unittest.skipUnless(torch.cuda.device_count() > 1	,     'test requires multiple GPUs'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			str   )   ->			List[str]:





						"""simple docstring"""






						return unittest.skipUnless(torch.xpu.device_count() > 1	,     'test requires multiple XPUs'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			str   )   ->			str:





						"""simple docstring"""






						return unittest.skipUnless(is_safetensors_available()	,     'test requires safetensors'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict   )   ->			List[str]:





						"""simple docstring"""






						return unittest.skipUnless(is_deepspeed_available()	,     'test requires DeepSpeed'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			int   )   ->			List[Any]:





						"""simple docstring"""






						return unittest.skipUnless(is_torch_version('>='	,     '1.12.0'   )	,     'test requires torch version >= 1.12.0'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict=None	,     _UpperCamelCase    :			Dict=None   )   ->			int:





						"""simple docstring"""






						if test_case is None:
												return partial(_UpperCamelCase	,     version=_UpperCamelCase   )
						return unittest.skipUnless(is_torch_version('>='	,     _UpperCamelCase   )	,     f"""test requires torch version >= {version}"""   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			Optional[int]   )   ->			List[str]:





						"""simple docstring"""






						return unittest.skipUnless(is_tensorboard_available()	,     'test requires Tensorboard'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			int   )   ->			int:





						"""simple docstring"""






						return unittest.skipUnless(is_wandb_available()	,     'test requires wandb'   )(_UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			int   )   ->			Any:





						"""simple docstring"""






						return unittest.skipUnless(is_comet_ml_available()	,     'test requires comet_ml'   )(_UpperCamelCase   )


SCREAMING_SNAKE_CASE__   =					(
    any([is_wandb_available(), is_tensorboard_available()]) and not is_comet_ml_available()
)





def 				lowerCAmelCase__      (     _UpperCamelCase    :			str   )   ->			List[str]:





						"""simple docstring"""






						return unittest.skipUnless(
						    _atleast_one_tracker_available	,     'test requires at least one tracker to be available and for `comet_ml` to not be installed'	,     )(_UpperCamelCase   )



class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  True

				@classmethod
				def 						snake_case	(			cls      ):






										"""simple docstring"""

										snake_case        =     tempfile.mkdtemp()

				@classmethod
				def 						snake_case	(			cls      ):






										"""simple docstring"""

										if os.path.exists(cls.tmpdir      ):
																shutil.rmtree(cls.tmpdir      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										if self.clear_on_setup:
																for path in Path(self.tmpdir      ).glob('**/*'      ):
																						if path.is_file():
																												path.unlink()
																						elif path.is_dir():
																												shutil.rmtree(lowerCAmelCase      )





class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""

				def 						snake_case	(			self      ):






										"""simple docstring"""

										super().tearDown()
										# Reset the state of the AcceleratorState singleton.
										AcceleratorState._reset_state()
										PartialState._reset_state()





class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""

				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     mocks if isinstance(lowerCAmelCase						,      (tuple, list)      ) else [mocks]
										for m in self.mocks:
																m.start()
																self.addCleanup(m.stop      )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			int   )   ->			Any:





						"""simple docstring"""






						snake_case        =     AcceleratorState()
						snake_case        =     tensor[None].clone().to(state.device   )
						snake_case        =     gather(_UpperCamelCase   ).cpu()
						snake_case        =     tensor[0].cpu()
						for i in range(tensors.shape[0]   ):
												if not torch.equal(tensors[i]	,     _UpperCamelCase   ):
																		return False
						return True



class 					lowerCAmelCase_      :



				"""simple docstring"""

				def __init__(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     returncode
										snake_case        =     stdout
										snake_case        =     stderr





async def 				lowerCAmelCase__      (     _UpperCamelCase    :			Dict	,     _UpperCamelCase    :			Any   )   ->			List[Any]:





						"""simple docstring"""






						while True:
												snake_case        =     await stream.readline()
												if line:
																		callback(_UpperCamelCase   )
												else:
																		break





async def 				lowerCAmelCase__      (     _UpperCamelCase    :			Optional[Any]	,     _UpperCamelCase    :			Optional[Any]=None	,     _UpperCamelCase    :			List[Any]=None	,     _UpperCamelCase    :			Optional[Any]=None	,     _UpperCamelCase    :			List[str]=False	,     _UpperCamelCase    :			Optional[int]=False   )   ->			_RunOutput:





						"""simple docstring"""






						if echo:
												print('\nRunning: '	,     ' '.join(_UpperCamelCase   )   )

						snake_case        =     await asyncio.create_subprocess_exec(
						    cmd[0]	,     *cmd[1:]	,     stdin=_UpperCamelCase	,     stdout=asyncio.subprocess.PIPE	,     stderr=asyncio.subprocess.PIPE	,     env=_UpperCamelCase	,     )

						# note: there is a warning for a possible deadlock when using `wait` with huge amounts of data in the pipe
						# https://docs.python.org/3/library/asyncio-subprocess.html#asyncio.asyncio.subprocess.Process.wait
						#
						# If it starts hanging, will need to switch to the following code. The problem is that no data
						# will be seen until it's done and if it hangs for example there will be no debug info.
						# out, err = await p.communicate()
						# return _RunOutput(p.returncode, out, err)

						snake_case        =     []
						snake_case        =     []

						def tee(_UpperCamelCase    :			List[Any]	,     _UpperCamelCase    :			Optional[Any]	,     _UpperCamelCase    :			Union[str, Any]	,     _UpperCamelCase    :			List[str]=""   ):
												snake_case        =     line.decode('utf-8'   ).rstrip()
												sink.append(_UpperCamelCase   )
												if not quiet:
																		print(_UpperCamelCase	,     _UpperCamelCase	,     file=_UpperCamelCase   )

    # XXX: the timeout doesn't seem to make any difference here
						await asyncio.wait(
						    [
						        asyncio.create_task(_read_stream(p.stdout	,     lambda _UpperCamelCase   : tee(_UpperCamelCase	,     _UpperCamelCase	,     sys.stdout	,     label='stdout:'   )   )   ),
						        asyncio.create_task(_read_stream(p.stderr	,     lambda _UpperCamelCase   : tee(_UpperCamelCase	,     _UpperCamelCase	,     sys.stderr	,     label='stderr:'   )   )   ),
						    ]	,     timeout=_UpperCamelCase	,     )
						return _RunOutput(await p.wait()	,     _UpperCamelCase	,     _UpperCamelCase   )





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[Any]	,     _UpperCamelCase    :			str=None	,     _UpperCamelCase    :			List[str]=None	,     _UpperCamelCase    :			Tuple=1_8_0	,     _UpperCamelCase    :			Dict=False	,     _UpperCamelCase    :			Optional[Any]=True   )   ->			_RunOutput:





						"""simple docstring"""






						snake_case        =     asyncio.get_event_loop()
						snake_case        =     loop.run_until_complete(
						    _stream_subprocess(_UpperCamelCase	,     env=_UpperCamelCase	,     stdin=_UpperCamelCase	,     timeout=_UpperCamelCase	,     quiet=_UpperCamelCase	,     echo=_UpperCamelCase   )   )

						snake_case        =     ' '.join(_UpperCamelCase   )
						if result.returncode > 0:
												snake_case        =     '\n'.join(result.stderr   )
												raise RuntimeError(
												    f"""'{cmd_str}' failed with returncode {result.returncode}\n\n"""
												    f"""The combined stderr from workers follows:\n{stderr}"""   )

						return result



class 					lowerCAmelCase_      (     lowerCAmelCase   ):



				"""simple docstring"""
				pass





def 				lowerCAmelCase__      (     _UpperCamelCase    :			List[str]	,     _UpperCamelCase    :			Optional[Any]=False   )   ->			Optional[Any]:





						"""simple docstring"""






						try:
												snake_case        =     subprocess.check_output(_UpperCamelCase	,     stderr=subprocess.STDOUT   )
												if return_stdout:
																		if hasattr(_UpperCamelCase	,     'decode'   ):
																								snake_case        =     output.decode('utf-8'   )
																		return output
						except subprocess.CalledProcessError as e:
												raise SubprocessCallException(
												    f"""Command `{" ".join(_UpperCamelCase   )}` failed with the following error:\n\n{e.output.decode()}"""   ) from e


