



import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length
def 		UpperCamelCase     (       snake_case__	:   Dict     )     ->					List[Any]:
   UpperCamelCase   :  Optional[Any]						     =     int(snake_case__     )
   UpperCamelCase     ,					UpperCamelCase     ,					UpperCamelCase   :  int						     =     t // 3600, (t // 60) % 60, t % 60
   return F"""{h}:{m:02d}:{s:02d}""" if h != 0 else F"""{m:02d}:{s:02d}"""
def 		UpperCamelCase     (       snake_case__	:   Union[str, Any]	,					snake_case__	:   List[Any]	,					snake_case__	:   int	,					snake_case__	:   Optional[int]	,					snake_case__	:   List[str]=300     )     ->					Optional[int]:
   # docstyle-ignore
   return F"""
    <div>
      {prefix}
      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>
      {label}
    </div>
    """
def 		UpperCamelCase     (       snake_case__	:   Any     )     ->					List[Any]:
   UpperCamelCase   :  Dict						     =     '<table border="1" class="dataframe">\n'
   html_code += """  <thead>\n <tr style="text-align: left;">\n"""
   for i in items[0]:
      html_code += F"""      <th>{i}</th>\n"""
   html_code += "    </tr>\n  </thead>\n  <tbody>\n"
   for line in items[1:]:
      html_code += "    <tr>\n"
      for elt in line:
         UpperCamelCase   :  List[str]						     =     F"""{elt:.6f}""" if isinstance(snake_case__	,					snake_case__     ) else str(snake_case__     )
         html_code += F"""      <td>{elt}</td>\n"""
      html_code += "    </tr>\n"
   html_code += "  </tbody>\n</table><p>"
   return html_code



class     lowerCAmelCase_		:
       UpperCAmelCase__    : str       					=  5
       UpperCAmelCase__    : Dict       					=  0.2
       def __init__(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = None,      SCREAMING_SNAKE_CASE_ = True,      SCREAMING_SNAKE_CASE_ = None,      SCREAMING_SNAKE_CASE_ = 300,      )					->		Tuple:
          UpperCamelCase   :  Dict						     =     total
          UpperCamelCase   :  Union[str, Any]						     =     '' if prefix is None else prefix
          UpperCamelCase   :  Optional[int]						     =     leave
          UpperCamelCase   :  Any						     =     parent
          UpperCamelCase   :  List[Any]						     =     width
          UpperCamelCase   :  int						     =     None
          UpperCamelCase   :  int						     =     None
          UpperCamelCase   :  Dict						     =     None
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_ = False,      SCREAMING_SNAKE_CASE_ = None	)					->		Optional[int]:
          UpperCamelCase   :  Optional[int]						     =     value
          if comment is not None:
             UpperCamelCase   :  Any						     =     comment
          if self.last_value is None:
             UpperCamelCase   :  str						     =     time.time()
             UpperCamelCase   :  str						     =     value
             UpperCamelCase   :  List[str]						     =     None
             UpperCamelCase   :  Optional[int]						     =     self.warmup
             UpperCamelCase   :  Optional[Any]						     =     1
             self.update_bar(SCREAMING_SNAKE_CASE_	)
          elif value <= self.last_value and not force_update:
             return
          elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for,      self.total	):
             if self.first_calls > 0:
                self.first_calls -= 1
             UpperCamelCase   :  List[str]						     =     time.time()
             UpperCamelCase   :  str						     =     current_time - self.start_time
             # We could have value = self.start_value if the update is called twixe with the same start value.
             if value > self.start_value:
                UpperCamelCase   :  Optional[int]						     =     self.elapsed_time / (value - self.start_value)
             else:
                UpperCamelCase   :  List[Any]						     =     None
             if value >= self.total:
                UpperCamelCase   :  Any						     =     self.total
                UpperCamelCase   :  Dict						     =     None
                if not self.leave:
                   self.close()
             elif self.average_time_per_item is not None:
                UpperCamelCase   :  List[Any]						     =     self.average_time_per_item * (self.total - value)
             self.update_bar(SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  int						     =     value
             UpperCamelCase   :  Tuple						     =     current_time
             if self.average_time_per_item is None:
                UpperCamelCase   :  int						     =     1
             else:
                UpperCamelCase   :  Optional[Any]						     =     max(int(self.update_every / self.average_time_per_item	),      1	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None	)					->		List[str]:
          UpperCamelCase   :  List[Any]						     =     ' ' * (len(str(self.total	)	) - len(str(SCREAMING_SNAKE_CASE_	)	)) + str(SCREAMING_SNAKE_CASE_	)
          if self.elapsed_time is None:
             UpperCamelCase   :  Optional[int]						     =     F"""[{spaced_value}/{self.total} : < :"""
          elif self.predicted_remaining is None:
             UpperCamelCase   :  int						     =     F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time	)}"""
          else:
             UpperCamelCase   :  Any						     =     (
                 F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time	)} <"""
                 F""" {format_time(self.predicted_remaining	)}"""
             )
             self.label += F""", {1/self.average_time_per_item:.2f} it/s"""
          self.label += "]" if self.comment is None or len(self.comment	) == 0 else F""", {self.comment}]"""
          self.display()
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  int						     =     html_progress_bar(self.value,      self.total,      self.prefix,      self.label,      self.width	)
          if self.parent is not None:
             # If this is a child bar, the parent will take care of the display.
             self.parent.display()
             return
          if self.output is None:
             UpperCamelCase   :  Optional[int]						     =     disp.display(disp.HTML(self.html_code	),      display_id=SCREAMING_SNAKE_CASE_	)
          else:
             self.output.update(disp.HTML(self.html_code	)	)






       def  snake_case_	(      self	)					->		Dict:
          if self.parent is None and self.output is not None:
             self.output.update(disp.HTML(''	)	)






class     lowerCAmelCase_		(  a__							):
       def __init__(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None	)					->		Optional[Any]:
          super().__init__(SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  str						     =     None if column_names is None else [column_names]
          UpperCamelCase   :  Any						     =     None
       def  snake_case_	(      self	)					->		List[str]:
          UpperCamelCase   :  Optional[Any]						     =     html_progress_bar(self.value,      self.total,      self.prefix,      self.label,      self.width	)
          if self.inner_table is not None:
             self.html_code += text_to_html_table(self.inner_table	)
          if self.child_bar is not None:
             self.html_code += self.child_bar.html_code
          if self.output is None:
             UpperCamelCase   :  Union[str, Any]						     =     disp.display(disp.HTML(self.html_code	),      display_id=SCREAMING_SNAKE_CASE_	)
          else:
             self.output.update(disp.HTML(self.html_code	)	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_	)					->		List[str]:
          if self.inner_table is None:
             UpperCamelCase   :  List[str]						     =     [list(values.keys()	), list(values.values()	)]
          else:
             UpperCamelCase   :  int						     =     self.inner_table[0]
             if len(self.inner_table	) == 1:
                # We give a chance to update the column names at the first iteration
                for key in values.keys():
                   if key not in columns:
                      columns.append(SCREAMING_SNAKE_CASE_	)
                UpperCamelCase   :  List[str]						     =     columns
             self.inner_table.append([values[c] for c in columns]	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=300	)					->		Tuple:
          UpperCamelCase   :  Any						     =     NotebookProgressBar(SCREAMING_SNAKE_CASE_,      prefix=SCREAMING_SNAKE_CASE_,      parent=self,      width=SCREAMING_SNAKE_CASE_	)
          return self.child_bar






       def  snake_case_	(      self	)					->		Any:
          UpperCamelCase   :  Optional[Any]						     =     None
          self.display()






class     lowerCAmelCase_		(  a__							):
       def __init__(      self	)					->		int:
          UpperCamelCase   :  List[Any]						     =     None
          UpperCamelCase   :  int						     =     None
          UpperCamelCase   :  Union[str, Any]						     =     False
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		Tuple:
          UpperCamelCase   :  Dict						     =     'Epoch' if args.evaluation_strategy == IntervalStrategy.EPOCH else 'Step'
          UpperCamelCase   :  Optional[int]						     =     0
          UpperCamelCase   :  Optional[int]						     =     0
          UpperCamelCase   :  str						     =     [self.first_column] + ['Training Loss']
          if args.evaluation_strategy != IntervalStrategy.NO:
             column_names.append('Validation Loss'	)
          UpperCamelCase   :  List[Any]						     =     NotebookTrainingTracker(state.max_steps,      SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		Dict:
          UpperCamelCase   :  List[str]						     =     int(state.epoch	) if int(state.epoch	) == state.epoch else F"""{state.epoch:.2f}"""
          self.training_tracker.update(
              state.global_step + 1,      comment=F"""Epoch {epoch}/{state.num_train_epochs}""",      force_update=self._force_next_update,      )
          UpperCamelCase   :  Optional[Any]						     =     False
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None,      **SCREAMING_SNAKE_CASE_	)					->		List[Any]:
          if not has_length(SCREAMING_SNAKE_CASE_	):
             return
          if self.prediction_bar is None:
             if self.training_tracker is not None:
                UpperCamelCase   :  Tuple						     =     self.training_tracker.add_child(len(SCREAMING_SNAKE_CASE_	)	)
             else:
                UpperCamelCase   :  Any						     =     NotebookProgressBar(len(SCREAMING_SNAKE_CASE_	)	)
             self.prediction_bar.update(1	)
          else:
             self.prediction_bar.update(self.prediction_bar.value + 1	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		str:
          if self.prediction_bar is not None:
             self.prediction_bar.close()
          UpperCamelCase   :  Optional[Any]						     =     None
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None,      **SCREAMING_SNAKE_CASE_	)					->		Any:
          # Only for when there is no evaluation
          if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
             UpperCamelCase   :  int						     =     {'Training Loss': logs['loss']}
             # First column is necessarily Step sine we're not in epoch eval strategy
             UpperCamelCase   :  List[Any]						     =     state.global_step
             self.training_tracker.write_line(SCREAMING_SNAKE_CASE_	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None,      **SCREAMING_SNAKE_CASE_	)					->		str:
          if self.training_tracker is not None:
             UpperCamelCase   :  List[str]						     =     {'Training Loss': 'No log', 'Validation Loss': 'No log'}
             for log in reversed(state.log_history	):
                if "loss" in log:
                   UpperCamelCase   :  List[Any]						     =     log['loss']
                   break

             if self.first_column == "Epoch":
                UpperCamelCase   :  Any						     =     int(state.epoch	)
             else:
                UpperCamelCase   :  Optional[int]						     =     state.global_step
             UpperCamelCase   :  List[str]						     =     'eval'
             for k in metrics:
                if k.endswith('_loss'	):
                   UpperCamelCase   :  int						     =     re.sub(R'\_loss$',      '',      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  Union[str, Any]						     =     metrics.pop('total_flos',      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  Any						     =     metrics.pop('epoch',      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  int						     =     metrics.pop(F"""{metric_key_prefix}_runtime""",      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  List[Any]						     =     metrics.pop(F"""{metric_key_prefix}_samples_per_second""",      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  List[str]						     =     metrics.pop(F"""{metric_key_prefix}_steps_per_second""",      SCREAMING_SNAKE_CASE_	)
             UpperCamelCase   :  str						     =     metrics.pop(F"""{metric_key_prefix}_jit_compilation_time""",      SCREAMING_SNAKE_CASE_	)
             for k, v in metrics.items():
                if k == F"""{metric_key_prefix}_loss""":
                   UpperCamelCase   :  Tuple						     =     v
                else:
                   UpperCamelCase   :  List[str]						     =     k.split('_'	)
                   UpperCamelCase   :  Optional[Any]						     =     ' '.join([part.capitalize() for part in splits[1:]]	)
                   UpperCamelCase   :  Optional[int]						     =     v
             self.training_tracker.write_line(SCREAMING_SNAKE_CASE_	)
             self.training_tracker.remove_child()
             UpperCamelCase   :  Any						     =     None
             # Evaluation takes a long time so we should force the next update.
             UpperCamelCase   :  Optional[int]						     =     True






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      **SCREAMING_SNAKE_CASE_	)					->		int:
          self.training_tracker.update(
              state.global_step,      comment=F"""Epoch {int(state.epoch	)}/{state.num_train_epochs}""",      force_update=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  Dict						     =     None








import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch



class     lowerCAmelCase_		(  unittest.TestCase							):
       UpperCAmelCase__    : Union[str, Any]       					=  JukeboxTokenizer
       UpperCAmelCase__    : Optional[int]       					=  {
           "artist": "Zac Brown Band",
           "genres": "Country",
           "lyrics": "I met a traveller from an antique land,\n        Who said \"Two vast and trunkless legs of stone\n        Stand in the desert. . . . Near them, on the sand,\n        Half sunk a shattered visage lies, whose frown,\n        And wrinkled lip, and sneer of cold command,\n        Tell that its sculptor well those passions read\n        Which yet survive, stamped on these lifeless things,\n        The hand that mocked them, and the heart that fed;\n        And on the pedestal, these words appear:\n        My name is Ozymandias, King of Kings;\n        Look on my Works, ye Mighty, and despair!\n        Nothing beside remains. Round the decay\n        Of that colossal Wreck, boundless and bare\n        The lone and level sands stretch far away\n        ",
       }
       @require_torch
       def  snake_case_	(      self	)					->		Optional[Any]:
          import torch

          UpperCamelCase   :  Tuple						     =     JukeboxTokenizer.from_pretrained('openai/jukebox-1b-lyrics'	)
          UpperCamelCase   :  List[str]						     =     tokenizer(**self.metas	)['input_ids']
          # fmt: off
          UpperCamelCase   :  Dict						     =     [
              torch.tensor([[
                  0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
                  76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
                  44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
                  47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
                  76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
                  30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
                  27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
                  45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
                  41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
                  76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
                  76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
                  64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
                  30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
                  27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
                  34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
                  27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
                  41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
                  44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
                  76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
                  32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
                  40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
                  45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
                  31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
                  45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
                  34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
                  31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
                  40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
                  38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
                  76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
                  41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
                  27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
                  46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
                  41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
                  46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
                  41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
                  40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
                  27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
                  76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
                  41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
                  76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
                  27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
                  34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
                  44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
                  40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
                  46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
                  38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
                  40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
                  27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
                  76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
                  76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
                  76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
                  76, 76]]	),
              torch.tensor([[0, 0, 0, 1069, 11]]	),
              torch.tensor([[0, 0, 0, 1069, 11]]	),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0],      EXPECTED_OUTPUT[0]	)	)
          self.assertTrue(torch.allclose(tokens[1],      EXPECTED_OUTPUT[1]	)	)
          self.assertTrue(torch.allclose(tokens[2],      EXPECTED_OUTPUT[2]	)	)






       @require_torch
       def  snake_case_	(      self	)					->		Optional[Any]:
          import torch

          UpperCamelCase   :  str						     =     JukeboxTokenizer.from_pretrained('openai/jukebox-5b-lyrics'	)
          UpperCamelCase   :  Dict						     =     tokenizer(**self.metas	)['input_ids']
          # fmt: off
          UpperCamelCase   :  Optional[int]						     =     [
              torch.tensor([[
                  0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
                  31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
                  31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
                  40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
                  79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
                  77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
                  27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
                  37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
                  32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
                  77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
                  77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
                  46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
                  77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
                  77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
                  77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
                  77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
                  64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
                  40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
                  40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
                  38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
                  31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
                  41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
                  46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
                  41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
                  31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
                  31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
                  44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
                  31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
                  38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
                  40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
                  27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
                  31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
                  34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
                  31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
                  31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
                  45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
                  31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
                  15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
                  11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
                  45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
                  41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
                  44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
                  46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
                  27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
                  35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
                  77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
                  31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
                  41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
                  77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
                  40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
                  77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
                  27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
                  77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
                  77, 77, 77, 77, 77, 77]]	),
              torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]	),
              torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]	),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0],      EXPECTED_OUTPUT[0]	)	)
          self.assertTrue(torch.allclose(tokens[1],      EXPECTED_OUTPUT[1]	)	)
          self.assertTrue(torch.allclose(tokens[2],      EXPECTED_OUTPUT[2]	)	)



