



'''simple docstring'''




import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCamelCase							:Tuple       = '''http://www.mocksite.com/file1.txt'''
lowerCamelCase							:str       = '''"text": ["foo", "foo"]'''
lowerCamelCase							:Optional[int]       = '''6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'''







class      _lowerCAmelCase							:
	__SCREAMING_SNAKE_CASE       :		List[Any]		      = 200
	__SCREAMING_SNAKE_CASE       :		Any		      = {'Content-Length': '100'}
	__SCREAMING_SNAKE_CASE       :		Dict		      = {}


	def    _a     (self	,				**lowercase							):
				return [bytes(lowercase	,				"""utf-8"""							)]






def      a      (			*lowerCamelCase__						,				**lowerCamelCase__   ):
			'''simple docstring'''





			return MockResponse()



@pytest.mark.parametrize("""urls_type"""						,				[str, list, dict]   )
def      a      (			lowerCamelCase__						,				lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			import requests

			monkeypatch.setattr(lowerCamelCase__						,				"""request"""						,				lowerCamelCase__   )

			A_     :						Tuple     =					URL
			if issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						Any     =					url
			elif issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						int     =					[url]
			elif issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						Optional[int]     =					{"""train""": url}
			A_     :						Union[str, Any]     =					"""dummy"""
			A_     :						str     =					"""downloads"""
			A_     :						Optional[int]     =					tmp_path
			A_     :						str     =					DownloadConfig(
			    cache_dir=os.path.join(lowerCamelCase__						,				lowerCamelCase__   )						,				use_etag=lowerCamelCase__						,				)
			A_     :						int     =					DownloadManager(dataset_name=lowerCamelCase__						,				download_config=lowerCamelCase__   )
			A_     :						Tuple     =					dl_manager.download(lowerCamelCase__   )
			A_     :						Any     =					urls
			for downloaded_paths in [downloaded_paths]:
						if isinstance(lowerCamelCase__						,				lowerCamelCase__   ):
									A_     :						Optional[int]     =					[downloaded_paths]
									A_     :						Union[str, Any]     =					[urls]
						elif isinstance(lowerCamelCase__						,				lowerCamelCase__   ):
									assert "train" in downloaded_paths.keys()
									A_     :						Any     =					downloaded_paths.values()
									A_     :						int     =					urls.values()
						assert downloaded_paths
						for downloaded_path, input_url in zip(lowerCamelCase__						,				lowerCamelCase__   ):
									assert downloaded_path == dl_manager.downloaded_paths[input_url]
									A_     :						int     =					Path(lowerCamelCase__   )
									A_     :						List[str]     =					downloaded_path.parts
									assert parts[-1] == HASH
									assert parts[-2] == cache_subdir
									assert downloaded_path.exists()
									A_     :						List[str]     =					downloaded_path.read_text()
									assert content == CONTENT
									A_     :						Union[str, Any]     =					downloaded_path.with_suffix(""".json"""   )
									assert metadata_downloaded_path.exists()
									A_     :						List[Any]     =					json.loads(metadata_downloaded_path.read_text()   )
									assert metadata_content == {"url": URL, "etag": None}



@pytest.mark.parametrize("""paths_type"""						,				[str, list, dict]   )
def      a      (			lowerCamelCase__						,				lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						Union[str, Any]     =					str(lowerCamelCase__   )
			if issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						str     =					filename
			elif issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						Optional[int]     =					[filename]
			elif issubclass(lowerCamelCase__						,				lowerCamelCase__   ):
						A_     :						str     =					{"""train""": filename}
			A_     :						int     =					"""dummy"""
			A_     :						Tuple     =					xz_file.parent
			A_     :						Optional[Any]     =					"""extracted"""
			A_     :						Union[str, Any]     =					DownloadConfig(
			    cache_dir=lowerCamelCase__						,				use_etag=lowerCamelCase__						,				)
			A_     :						Dict     =					DownloadManager(dataset_name=lowerCamelCase__						,				download_config=lowerCamelCase__   )
			A_     :						List[Any]     =					dl_manager.extract(lowerCamelCase__   )
			A_     :						Optional[Any]     =					paths
			for extracted_paths in [extracted_paths]:
						if isinstance(lowerCamelCase__						,				lowerCamelCase__   ):
									A_     :						Tuple     =					[extracted_paths]
									A_     :						int     =					[paths]
						elif isinstance(lowerCamelCase__						,				lowerCamelCase__   ):
									assert "train" in extracted_paths.keys()
									A_     :						int     =					extracted_paths.values()
									A_     :						int     =					paths.values()
						assert extracted_paths
						for extracted_path, input_path in zip(lowerCamelCase__						,				lowerCamelCase__   ):
									assert extracted_path == dl_manager.extracted_paths[input_path]
									A_     :						Dict     =					Path(lowerCamelCase__   )
									A_     :						Tuple     =					extracted_path.parts
									assert parts[-1] == hash_url_to_filename(lowerCamelCase__						,				etag=lowerCamelCase__   )
									assert parts[-2] == extracted_subdir
									assert extracted_path.exists()
									A_     :						int     =					extracted_path.read_text()
									A_     :						Optional[int]     =					text_file.read_text()
									assert extracted_file_content == expected_file_content



def      a      (			lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			assert path.endswith(""".jsonl"""   )
			for num_items, line in enumerate(lowerCamelCase__						,				start=1   ):
						A_     :						Optional[Any]     =					json.loads(line.decode("""utf-8"""   )   )
						assert item.keys() == {"col_1", "col_2", "col_3"}
			assert num_items == 4



@pytest.mark.parametrize("""archive_jsonl"""						,				["""tar_jsonl_path""", """zip_jsonl_path"""]   )
def      a      (			lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						Tuple     =					request.getfixturevalue(lowerCamelCase__   )
			A_     :						List[str]     =					DownloadManager()
			for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowerCamelCase__   )						,				start=1   ):
						_test_jsonl(lowerCamelCase__						,				lowerCamelCase__   )
			assert num_jsonl == 2



@pytest.mark.parametrize("""archive_nested_jsonl"""						,				["""tar_nested_jsonl_path""", """zip_nested_jsonl_path"""]   )
def      a      (			lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						Tuple     =					request.getfixturevalue(lowerCamelCase__   )
			A_     :						Optional[int]     =					DownloadManager()
			for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowerCamelCase__   )						,				start=1   ):
						for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowerCamelCase__   )						,				start=1   ):
									_test_jsonl(lowerCamelCase__						,				lowerCamelCase__   )
			assert num_tar == 1
			assert num_jsonl == 2



def      a      (			lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						Dict     =					DownloadManager()
			for num_file, file in enumerate(dl_manager.iter_files(lowerCamelCase__   )						,				start=1   ):
						assert os.path.basename(lowerCamelCase__   ) == ("test.txt" if num_file == 1 else "train.txt")
			assert num_file == 2




'''simple docstring'''




from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    flip_channel_order,
    get_resize_output_image_size,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, is_vision_available, logging


if is_vision_available():
						import PIL

if is_torch_available():
						import torch


lowerCamelCase							:str       = logging.get_logger(__name__)







class      _lowerCAmelCase							(      __UpperCAmelCase			):
	__SCREAMING_SNAKE_CASE       :		Tuple		      = ['pixel_values']


	def __init__(self	,				lowercase = True	,				lowercase = None	,				lowercase = PILImageResampling.BILINEAR	,				lowercase = True	,				lowercase = 1 / 255	,				lowercase = True	,				lowercase = None	,				lowercase = True	,				**lowercase	,				):
				super().__init__(**lowercase							)
				A_     :						Union[str, Any]     =					size if size is not None else {"""shortest_edge""": 224}
				A_     :						Union[str, Any]     =					get_size_dict(lowercase	,				default_to_square=lowercase							)
				A_     :						Optional[Any]     =					crop_size if crop_size is not None else {"""height""": 256, """width""": 256}
				A_     :						Tuple     =					get_size_dict(lowercase	,				param_name="""crop_size"""							)

				A_     :						List[Any]     =					do_resize
				A_     :						List[str]     =					size
				A_     :						Dict     =					resample
				A_     :						int     =					do_rescale
				A_     :						str     =					rescale_factor
				A_     :						Tuple     =					do_center_crop
				A_     :						Tuple     =					crop_size
				A_     :						List[str]     =					do_flip_channel_order


	def    _a     (self	,				lowercase	,				lowercase	,				lowercase = PIL.Image.BILINEAR	,				lowercase = None	,				**lowercase	,				):
				A_     :						Any     =					get_size_dict(lowercase	,				default_to_square=lowercase							)
				if "shortest_edge" not in size:
							raise ValueError(F'The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}'							)
				A_     :						Any     =					get_resize_output_image_size(lowercase	,				size=size["""shortest_edge"""]	,				default_to_square=lowercase							)
				return resize(lowercase	,				size=lowercase	,				resample=lowercase	,				data_format=lowercase	,				**lowercase							)


	def    _a     (self	,				lowercase	,				lowercase	,				lowercase = None	,				**lowercase	,				):
				A_     :						Tuple     =					get_size_dict(lowercase							)
				if "height" not in size or "width" not in size:
							raise ValueError(F'The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}'							)
				return center_crop(lowercase	,				size=(size["""height"""], size["""width"""])	,				data_format=lowercase	,				**lowercase							)


	def    _a     (self	,				lowercase	,				lowercase	,				lowercase = None	,				**lowercase	,				):
				return rescale(lowercase	,				scale=lowercase	,				data_format=lowercase	,				**lowercase							)


	def    _a     (self	,				lowercase	,				lowercase = None							):
				return flip_channel_order(lowercase	,				data_format=lowercase							)


	def    _a     (self	,				lowercase	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = None	,				lowercase = ChannelDimension.FIRST	,				**lowercase	,				):
				A_     :						str     =					do_resize if do_resize is not None else self.do_resize
				A_     :						Optional[int]     =					resample if resample is not None else self.resample
				A_     :						str     =					do_rescale if do_rescale is not None else self.do_rescale
				A_     :						str     =					rescale_factor if rescale_factor is not None else self.rescale_factor
				A_     :						str     =					do_center_crop if do_center_crop is not None else self.do_center_crop
				A_     :						Dict     =					(
				    do_flip_channel_order if do_flip_channel_order is not None else self.do_flip_channel_order
				)

				A_     :						Union[str, Any]     =					size if size is not None else self.size
				A_     :						Dict     =					get_size_dict(lowercase	,				default_to_square=lowercase							)
				A_     :						Any     =					crop_size if crop_size is not None else self.crop_size
				A_     :						Union[str, Any]     =					get_size_dict(lowercase	,				param_name="""crop_size"""							)

				A_     :						Union[str, Any]     =					make_list_of_images(lowercase							)

				if not valid_images(lowercase							):
							raise ValueError(
							    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
							    """torch.Tensor, tf.Tensor or jax.ndarray."""							)

				if do_resize and size is None:
							raise ValueError("""Size must be specified if do_resize is True."""							)

				if do_rescale and rescale_factor is None:
							raise ValueError("""Rescale factor must be specified if do_rescale is True."""							)

				if do_center_crop and crop_size is None:
							raise ValueError("""Crop size must be specified if do_center_crop is True."""							)

				# All transformations expect numpy arrays.
				A_     :						Optional[Any]     =					[to_numpy_array(lowercase							) for image in images]

				if do_resize:
							A_     :						List[Any]     =					[self.resize(image=lowercase	,				size=lowercase	,				resample=lowercase							) for image in images]

				if do_center_crop:
							A_     :						str     =					[self.center_crop(image=lowercase	,				size=lowercase							) for image in images]

				if do_rescale:
							A_     :						Dict     =					[self.rescale(image=lowercase	,				scale=lowercase							) for image in images]

				# the pretrained checkpoints assume images are BGR, not RGB
				if do_flip_channel_order:
							A_     :						List[str]     =					[self.flip_channel_order(image=lowercase							) for image in images]

				A_     :						List[str]     =					[to_channel_dimension_format(lowercase	,				lowercase							) for image in images]

				A_     :						str     =					{"""pixel_values""": images}
				return BatchFeature(data=lowercase	,				tensor_type=lowercase							)


	def    _a     (self	,				lowercase	,				lowercase = None							):
				A_     :						Optional[Any]     =					outputs.logits

				# Resize logits and compute semantic segmentation maps
				if target_sizes is not None:
							if len(lowercase							) != len(lowercase							):
										raise ValueError(
										    """Make sure that you pass in as many target sizes as the batch dimension of the logits"""							)

							if is_torch_tensor(lowercase							):
										A_     :						Dict     =					target_sizes.numpy()

							A_     :						Union[str, Any]     =					[]

							for idx in range(len(lowercase							)							):
										A_     :						Union[str, Any]     =					torch.nn.functional.interpolate(
										    logits[idx].unsqueeze(dim=0							)	,				size=target_sizes[idx]	,				mode="""bilinear"""	,				align_corners=lowercase							)
										A_     :						Any     =					resized_logits[0].argmax(dim=0							)
										semantic_segmentation.append(lowercase							)
				else:
							A_     :						str     =					logits.argmax(dim=1							)
							A_     :						Optional[Any]     =					[semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]							)]

				return semantic_segmentation