



'''simple docstring'''






import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)

class 							a__( lowerCamelCase__     ):







    def __init__(				self			:  int						,      __snake_case			:  Optional[Any]=0.01						,      __snake_case			:  Any=10_00     ):
      a						: Optional[int]					       =					p_stop
      a						: Tuple					       =					max_length







    def __iter__(				self			:  Union[str, Any]     ):
      a						: Optional[int]					       =					0
      a						: Tuple					       =					False
      while not stop and count < self.max_length:
        yield count
        count += 1
        a						: Optional[Any]					       =					random.random() < self.p_stop

class 							a__( unittest.TestCase     ):







    def       lowercase_	(				self			:  str						,      __snake_case			:  List[Any]						,      __snake_case			:  int						,      __snake_case			:  int=False						,      __snake_case			:  int=True     ):
      a						: Union[str, Any]					       =					[
          BatchSamplerShard(__snake_case						,      2						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )
          for i in range(2     )
      ]
      a						: Optional[Any]					       =					[list(__snake_case     ) for batch_sampler_shard in batch_sampler_shards]
      if not split_batches:
        self.assertListEqual([len(__snake_case     ) for shard in batch_sampler_shards]						,      [len(__snake_case     ) for e in expected]     )
      self.assertListEqual(__snake_case						,      __snake_case     )







    def       lowercase_	(				self			:  Any     ):
      # Check the shards when the dataset is a round multiple of total batch size.
      a						: Optional[int]					       =					BatchSampler(range(24     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Union[str, Any]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      a						: str					       =					BatchSampler(range(24     )						,      batch_size=3						,      drop_last=__snake_case     )
      # Expected shouldn't change
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      # Check the shards when the dataset is a round multiple of batch size but not total batch size.
      a						: Any					       =					BatchSampler(range(21     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Optional[Any]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      a						: Any					       =					BatchSampler(range(21     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: List[str]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
      # num_processes batch.
      a						: str					       =					BatchSampler(range(22     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Union[str, Any]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      a						: int					       =					BatchSampler(range(22     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: str					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
      # num_processes batch.
      a						: List[str]					       =					BatchSampler(range(20     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: List[str]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      a						: Union[str, Any]					       =					BatchSampler(range(20     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: List[Any]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      # Check the shards when the dataset is very small.
      a						: Dict					       =					BatchSampler(range(2     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Any					       =					[[[0, 1, 0]], [[1, 0, 1]]]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )

      a						: List[str]					       =					BatchSampler(range(2     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Any					       =					[[], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case     )







    def       lowercase_	(				self			:  Any     ):
      # Check the shards when the dataset is a round multiple of batch size.
      a						: Dict					       =					BatchSampler(range(24     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Any					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      a						: Optional[int]					       =					BatchSampler(range(24     )						,      batch_size=4						,      drop_last=__snake_case     )
      # Expected shouldn't change
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size.
      a						: Union[str, Any]					       =					BatchSampler(range(22     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: List[Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      a						: Any					       =					BatchSampler(range(22     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Union[str, Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size or num_processes.
      a						: Any					       =					BatchSampler(range(21     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: str					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      a						: List[str]					       =					BatchSampler(range(21     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: str					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      # Check the shards when the dataset is very small.
      a						: Optional[Any]					       =					BatchSampler(range(2     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Optional[int]					       =					[[[0, 1]], [[0, 1]]]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )

      a						: Dict					       =					BatchSampler(range(2     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: str					       =					[[], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case     )







    def       lowercase_	(				self			:  List[Any]     ):
      # Check the shards when the dataset is a round multiple of total batch size.
      a						: Union[str, Any]					       =					BatchSampler(range(24     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Tuple					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      a						: Any					       =					BatchSampler(range(24     )						,      batch_size=3						,      drop_last=__snake_case     )
      # Expected shouldn't change
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is a round multiple of batch size but not total batch size.
      a						: List[Any]					       =					BatchSampler(range(21     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: int					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      a						: str					       =					BatchSampler(range(21     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: List[Any]					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
      # num_processes batch.
      a						: Optional[int]					       =					BatchSampler(range(22     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Any					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      a						: Optional[int]					       =					BatchSampler(range(22     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: int					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
      # num_processes batch.
      a						: Tuple					       =					BatchSampler(range(20     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Any					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      a						: Union[str, Any]					       =					BatchSampler(range(20     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Tuple					       =					[
          [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
          [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is very small.
      a						: Dict					       =					BatchSampler(range(2     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Any					       =					[[[0, 1]], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )

      a						: List[Any]					       =					BatchSampler(range(2     )						,      batch_size=3						,      drop_last=__snake_case     )
      a						: Tuple					       =					[[], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      even_batches=__snake_case     )







    def       lowercase_	(				self			:  Union[str, Any]     ):
      # Check the shards when the dataset is a round multiple of batch size.
      a						: List[Any]					       =					BatchSampler(range(24     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Tuple					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      a						: int					       =					BatchSampler(range(24     )						,      batch_size=4						,      drop_last=__snake_case     )
      # Expected shouldn't change
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size.
      a						: List[Any]					       =					BatchSampler(range(22     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Optional[Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      a						: Optional[Any]					       =					BatchSampler(range(22     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Union[str, Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is not a round multiple of batch size or num_processes.
      a						: Tuple					       =					BatchSampler(range(21     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: List[Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      a						: Optional[Any]					       =					BatchSampler(range(21     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Optional[Any]					       =					[
          [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
          [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
      ]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      # Check the shards when the dataset is very small.
      a						: List[Any]					       =					BatchSampler(range(2     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: str					       =					[[[0, 1]], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )

      a						: int					       =					BatchSampler(range(2     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: List[Any]					       =					[[], []]
      self.check_batch_sampler_shards(__snake_case						,      __snake_case						,      split_batches=__snake_case						,      even_batches=__snake_case     )







    def       lowercase_	(				self			:  Any     ):
      a						: Optional[Any]					       =					[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
      a						: List[str]					       =					[BatchSamplerShard(__snake_case						,      2						,      __snake_case						,      even_batches=__snake_case     ) for i in range(2     )]

      self.assertEqual(len(batch_sampler_shards[0]     )						,      3     )
      self.assertEqual(len(batch_sampler_shards[1]     )						,      2     )

      self.assertListEqual(list(batch_sampler_shards[0]     )						,      [[0, 1, 2], [5, 6, 7, 8], [12, 13]]     )
      self.assertListEqual(list(batch_sampler_shards[1]     )						,      [[3, 4], [9, 10, 11]]     )







    def       lowercase_	(				self			:  Optional[int]						,      __snake_case			:  List[str]						,      __snake_case			:  Union[str, Any]						,      __snake_case			:  Dict						,      __snake_case			:  Union[str, Any]=False						,      __snake_case			:  List[str]=2						,      __snake_case			:  Optional[int]=False     ):
      random.seed(__snake_case     )
      a						: Optional[Any]					       =					list(__snake_case     )

      a						: Dict					       =					[
          IterableDatasetShard(
              __snake_case						,      batch_size=__snake_case						,      drop_last=__snake_case						,      num_processes=__snake_case						,      process_index=__snake_case						,      split_batches=__snake_case						,      )
          for i in range(__snake_case     )
      ]
      a						: Dict					       =					[]
      for iterable_dataset_shard in iterable_dataset_shards:
        # Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
        random.seed(__snake_case     )
        iterable_dataset_lists.append(list(__snake_case     )     )

      a						: Tuple					       =					batch_size // num_processes if split_batches else batch_size
      # All iterable dataset shard should have the same length, a round multiple of shard_batch_size
      a						: Union[str, Any]					       =					iterable_dataset_lists[0]
      for l in iterable_dataset_lists[1:]:
        self.assertEqual(len(__snake_case     )						,      len(__snake_case     )     )
        self.assertTrue(len(__snake_case     ) % shard_batch_size == 0     )

      a						: Tuple					       =					[]
      for idx in range(0						,      len(__snake_case     )						,      __snake_case     ):
        for l in iterable_dataset_lists:
          observed += l[idx : idx + shard_batch_size]

      if not drop_last:
        while len(__snake_case     ) < len(__snake_case     ):
          reference += reference
      self.assertListEqual(__snake_case						,      reference[: len(__snake_case     )]     )







    def       lowercase_	(				self			:  Tuple     ):
      a						: Optional[int]					       =					42
      a						: int					       =					RandomIterableDataset()

      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )

      # Edge case with a very small dataset
      a						: int					       =					RandomIterableDataset(max_length=2     )

      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )
      self.check_iterable_dataset_shards(__snake_case						,      __snake_case						,      batch_size=4						,      drop_last=__snake_case						,      split_batches=__snake_case     )







    def       lowercase_	(				self			:  Optional[int]     ):
      a						: Tuple					       =					BatchSampler(range(16     )						,      batch_size=4						,      drop_last=__snake_case     )
      a						: Optional[Any]					       =					SkipBatchSampler(__snake_case						,      2     )
      self.assertListEqual(list(__snake_case     )						,      [[8, 9, 10, 11], [12, 13, 14, 15]]     )







    def       lowercase_	(				self			:  int     ):
      a						: Optional[int]					       =					SkipDataLoader(list(range(16     )     )						,      batch_size=4						,      skip_batches=2     )
      self.assertListEqual([t.tolist() for t in dataloader]						,      [[8, 9, 10, 11], [12, 13, 14, 15]]     )







    def       lowercase_	(				self			:  Optional[Any]     ):
      a						: List[str]					       =					DataLoader(list(range(16     )     )						,      batch_size=4     )
      a						: int					       =					skip_first_batches(__snake_case						,      num_batches=2     )
      self.assertListEqual([t.tolist() for t in new_dataloader]						,      [[8, 9, 10, 11], [12, 13, 14, 15]]     )







    def       lowercase_	(				self			:  Any     ):
      a						: Dict					       =					DataLoaderShard(list(range(16     )     )						,      batch_size=4     )
      for idx, _ in enumerate(__snake_case     ):
        self.assertEqual(dataloader.end_of_dataloader						,      idx == 3     )

      # Test it also works on the second iteration
      for idx, _ in enumerate(__snake_case     ):
        self.assertEqual(dataloader.end_of_dataloader						,      idx == 3     )







    def       lowercase_	(				self			:  Union[str, Any]     ):
      Accelerator()
      a						: int					       =					DataLoaderDispatcher(range(16     )						,      batch_size=4     )
      for idx, _ in enumerate(__snake_case     ):
        self.assertEqual(dataloader.end_of_dataloader						,      idx == 3     )

      # Test it also works on the second iteration
      for idx, _ in enumerate(__snake_case     ):
        self.assertEqual(dataloader.end_of_dataloader						,      idx == 3     )




'''simple docstring'''






import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 							a__( lowerCamelCase__		,			unittest.TestCase     ):
    lowercase__			=   CTRLTokenizer
    lowercase__			=   False
    lowercase__			=   False







    def       lowercase_	(				self			:  Dict     ):
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      a						: Tuple					       =					['adapt', 're@@', 'a@@', 'apt', 'c@@', 't', '<unk>']
      a						: Union[str, Any]					       =					dict(zip(__snake_case						,      range(len(__snake_case     )     )     )     )
      a						: Union[str, Any]					       =					['#version: 0.2', 'a p', 'ap t</w>', 'r e', 'a d', 'ad apt</w>', '']
      a						: Optional[Any]					       =					{'unk_token': '<unk>'}

      a						: Dict					       =					os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['vocab_file']     )
      a						: Optional[Any]					       =					os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['merges_file']     )
      with open(self.vocab_file						,      'w'						,      encoding='utf-8'     ) as fp:
        fp.write(json.dumps(__snake_case     ) + '\n'     )
      with open(self.merges_file						,      'w'						,      encoding='utf-8'     ) as fp:
        fp.write('\n'.join(__snake_case     )     )







    def       lowercase_	(				self			:  int						,      **__snake_case			:  str     ):
      kwargs.update(self.special_tokens_map     )
      return CTRLTokenizer.from_pretrained(self.tmpdirname						,      **__snake_case     )







    def       lowercase_	(				self			:  Optional[int]						,      __snake_case			:  Any     ):
      a						: int					       =					'adapt react readapt apt'
      a						: Any					       =					'adapt react readapt apt'
      return input_text, output_text







    def       lowercase_	(				self			:  Dict     ):
      a						: Dict					       =					CTRLTokenizer(self.vocab_file						,      self.merges_file						,      **self.special_tokens_map     )
      a						: List[str]					       =					'adapt react readapt apt'
      a						: Dict					       =					'adapt re@@ a@@ c@@ t re@@ adapt apt'.split()
      a						: Any					       =					tokenizer.tokenize(__snake_case     )
      self.assertListEqual(__snake_case						,      __snake_case     )

      a						: Dict					       =					tokens + [tokenizer.unk_token]

      a						: Optional[Any]					       =					[0, 1, 2, 4, 5, 1, 0, 3, 6]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case     )						,      __snake_case     )