


"""simple docstring"""







import os
from collections import deque

import torch
from torch.utils.data import Dataset


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__=""		,					snake_case__="train"						):






         """simple docstring"""





         assert os.path.isdir(snake_case__						)

         lowerCAmelCase					:					Any								=							[]
         lowerCAmelCase					:					List[str]								=							os.listdir(snake_case__						)
         for story_filename in story_filenames_list:
           if "summary" in story_filename:
             continue
           lowerCAmelCase					:					int								=							os.path.join(snake_case__		,					snake_case__						)
           if not os.path.isfile(snake_case__						):
             continue
           self.documents.append(snake_case__						)

       def __len__(						self						):






         """simple docstring"""





         return len(self.documents						)







       def __getitem__(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							self.documents[idx]
         lowerCAmelCase					:					Optional[int]								=							document_path.split("/"						)[-1]
         with open(snake_case__		,					encoding="utf-8"						) as source:
           lowerCAmelCase					:					List[str]								=							source.read()
           lowerCAmelCase						,  lowerCAmelCase					:					str								=							process_story(snake_case__						)
         return document_name, story_lines, summary_lines

def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					str								=							list(filter(lambda SCREAMING_SNAKE_CASE    : len(SCREAMING_SNAKE_CASE    ) != 0    ,				[line.strip() for line in raw_story.split("\n"    )]    )    )

  # for some unknown reason some lines miss a period, add it
  lowerCAmelCase					:					Dict								=							[_add_missing_period(SCREAMING_SNAKE_CASE    ) for line in nonempty_lines]

  # gather article lines
  lowerCAmelCase					:					Optional[int]								=							[]
  lowerCAmelCase					:					List[Any]								=							deque(SCREAMING_SNAKE_CASE    )
  while True:
    try:
      lowerCAmelCase					:					Dict								=							lines.popleft()
      if element.startswith("@highlight"    ):
        break
      story_lines.append(SCREAMING_SNAKE_CASE    )
    except IndexError:
      # if "@highlight" is absent from the file we pop
      # all elements until there is None, raising an exception.
      return story_lines, []

    # gather summary lines
  lowerCAmelCase					:					Dict								=							list(filter(lambda SCREAMING_SNAKE_CASE    : not t.startswith("@highlight"    )    ,				SCREAMING_SNAKE_CASE    )    )

  return story_lines, summary_lines
def  a__      (    SCREAMING_SNAKE_CASE   :   int    ):


  '''simple docstring'''







  lowerCAmelCase					:					Dict								=							[".", "!", "?", "...", "'", "`", "\"", "\u2019", "\u2019", ")"]
  if line.startswith("@highlight"    ):
    return line
  if line[-1] in END_TOKENS:
    return line
  return line + "."
def  a__      (    SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   int    ,				SCREAMING_SNAKE_CASE   :   int    ):


  '''simple docstring'''







  if len(SCREAMING_SNAKE_CASE    ) > block_size:
    return sequence[:block_size]
  else:
    sequence.extend([pad_token_id] * (block_size - len(SCREAMING_SNAKE_CASE    ))    )
    return sequence
def  a__      (    SCREAMING_SNAKE_CASE   :   Dict    ,				SCREAMING_SNAKE_CASE   :   str    ):


  '''simple docstring'''







  lowerCAmelCase					:					str								=							torch.ones_like(SCREAMING_SNAKE_CASE    )
  lowerCAmelCase					:					Union[str, Any]								=							sequence == pad_token_id
  lowerCAmelCase					:					List[str]								=							0
  return mask
def  a__      (    SCREAMING_SNAKE_CASE   :   int    ,				SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   Optional[int]    ):


  '''simple docstring'''







  lowerCAmelCase					:					str								=							[tokenizer.encode(SCREAMING_SNAKE_CASE    ) for line in story_lines]
  lowerCAmelCase					:					Any								=							[token for sentence in story_lines_token_ids for token in sentence]
  lowerCAmelCase					:					Optional[int]								=							[tokenizer.encode(SCREAMING_SNAKE_CASE    ) for line in summary_lines]
  lowerCAmelCase					:					Union[str, Any]								=							[token for sentence in summary_lines_token_ids for token in sentence]

  return story_token_ids, summary_token_ids





def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ,				SCREAMING_SNAKE_CASE   :   Optional[Any]    ):


  '''simple docstring'''







  lowerCAmelCase					:					List[Any]								=							[]
  for sequence in batch:
    lowerCAmelCase					:					str								=							-1
    lowerCAmelCase					:					str								=							[]
    for s in sequence:
      if s == separator_token_id:
        sentence_num += 1
      embeddings.append(sentence_num % 2    )
    batch_embeddings.append(SCREAMING_SNAKE_CASE    )
  return torch.tensor(SCREAMING_SNAKE_CASE    )









"""simple docstring"""







import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible


class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
             "unet/diffusion_pytorch_model.bin",
             # Removed: 'unet/diffusion_pytorch_model.safetensors',
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         self.assertTrue(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							[
             "safety_checker/pytorch_model.bin",
             "safety_checker/model.safetensors",
             "vae/diffusion_pytorch_model.bin",
             "vae/diffusion_pytorch_model.safetensors",
             "text_encoder/pytorch_model.bin",
             # Removed: 'text_encoder/model.safetensors',
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         self.assertFalse(is_safetensors_compatible(snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							[
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "unet/diffusion_pytorch_model.bin",
             "unet/diffusion_pytorch_model.safetensors",
         ]
         lowerCAmelCase					:					Union[str, Any]								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
             "unet/diffusion_pytorch_model.fp16.bin",
             # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
         ]
         lowerCAmelCase					:					Dict								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.fp16.bin",
             "text_encoder/model.fp16.safetensors",
         ]
         lowerCAmelCase					:					int								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							[
             "text_encoder/pytorch_model.bin",
             "text_encoder/model.safetensors",
         ]
         lowerCAmelCase					:					Tuple								=							"fp16"
         self.assertTrue(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							[
             "safety_checker/pytorch_model.fp16.bin",
             "safety_checker/model.fp16.safetensors",
             "vae/diffusion_pytorch_model.fp16.bin",
             "vae/diffusion_pytorch_model.fp16.safetensors",
             "text_encoder/pytorch_model.fp16.bin",
             # 'text_encoder/model.fp16.safetensors',
             "unet/diffusion_pytorch_model.fp16.bin",
             "unet/diffusion_pytorch_model.fp16.safetensors",
         ]
         lowerCAmelCase					:					List[Any]								=							"fp16"
         self.assertFalse(is_safetensors_compatible(snake_case__		,					variant=snake_case__						)						)





