/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.api;

import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;

import org.apache.parquet.schema.MessageType;

/**
 *
 * Context passed to ReadSupport when initializing for read
 */
public class InitContext {

  private final Map<String,Set<String>> keyValueMetadata;
  private Map<String,String> mergedKeyValueMetadata;
  private final Configuration configuration;
  private final MessageType fileSchema;

  /**
   * @param configuration the hadoop configuration
   * @param keyValueMetadata extra metadata from file footers
   * @param fileSchema the merged schema from the files
   */
  public InitContext(
      Configuration configuration,
      Map<String, Set<String>> keyValueMetadata,
      MessageType fileSchema) {
    super();
    this.keyValueMetadata = keyValueMetadata;
    this.configuration = configuration;
    this.fileSchema = fileSchema;
  }

  /**
   * If there is a conflicting value when reading from multiple files,
   * an exception will be thrown
   * @return the merged key values metadata form the file footers
   */
  @Deprecated
  public Map<String, String> getMergedKeyValueMetaData() {
    if (mergedKeyValueMetadata == null) {
      Map<String, String> mergedKeyValues = new HashMap<String, String>();
      for (Entry<String, Set<String>> entry : keyValueMetadata.entrySet()) {
        if (entry.getValue().size() > 1) {
          throw new RuntimeException("could not merge metadata: key " + entry.getKey() + " has conflicting values: " + entry.getValue());
        }
        mergedKeyValues.put(entry.getKey(), entry.getValue().iterator().next());
      }
      mergedKeyValueMetadata = mergedKeyValues;
    }
    return mergedKeyValueMetadata;
  }

  /**
   * @return the configuration for this job
   */
  public Configuration getConfiguration() {
    return configuration;
  }

  /**
   * this is the union of all the schemas when reading multiple files.
   * @return the schema of the files being read
   */
  public MessageType getFileSchema() {
    return fileSchema;
  }

  /**
   * each key is associated with the list of distinct values found in footers
   * @return the merged metadata from the footer of the file
   */
  public Map<String, Set<String>> getKeyValueMetadata() {
    return keyValueMetadata;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hadoop.hive.ql.io.parquet.read;

import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;

import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;

import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetInputFormat;
import org.apache.parquet.hadoop.ParquetInputSplit;
import org.apache.parquet.hadoop.api.ReadSupport.ReadContext;
import org.apache.parquet.hadoop.metadata.FileMetaData;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.ContextUtil;
import org.apache.parquet.hive.HiveBinding;
import org.apache.parquet.hive.HiveBindingFactory;
import org.apache.parquet.schema.MessageTypeParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ParquetRecordReaderWrapper  implements RecordReader<Void, ArrayWritable> {
  public static final Logger LOG = LoggerFactory.getLogger(ParquetRecordReaderWrapper.class);

  private final long splitLen; // for getPos()

  private org.apache.hadoop.mapreduce.RecordReader<Void, ArrayWritable> realReader;
  // expect readReader return same Key & Value objects (common case)
  // this avoids extra serialization & deserialization of these objects
  private ArrayWritable valueObj = null;
  private boolean firstRecord = false;
  private boolean eof = false;
  private int schemaSize;

  private final HiveBinding hiveBinding;

  public ParquetRecordReaderWrapper(
      final ParquetInputFormat<ArrayWritable> newInputFormat,
      final InputSplit oldSplit,
      final JobConf oldJobConf,
      final Reporter reporter)
          throws IOException, InterruptedException {
    this(newInputFormat, oldSplit, oldJobConf, reporter,
        (new HiveBindingFactory()).create());
  }

  public ParquetRecordReaderWrapper(
      final ParquetInputFormat<ArrayWritable> newInputFormat,
      final InputSplit oldSplit,
      final JobConf oldJobConf,
      final Reporter reporter,
      final HiveBinding hiveBinding)
          throws IOException, InterruptedException {
    this.splitLen = oldSplit.getLength();
    this.hiveBinding = hiveBinding;

    final ParquetInputSplit split = getSplit(oldSplit, oldJobConf);

    TaskAttemptID taskAttemptID = TaskAttemptID.forName(oldJobConf.get(IOConstants.MAPRED_TASK_ID));
    if (taskAttemptID == null) {
      taskAttemptID = new TaskAttemptID();
    }

    // create a TaskInputOutputContext
    final TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(oldJobConf, taskAttemptID);

    if (split != null) {
      try {
        realReader = newInputFormat.createRecordReader(split, taskContext);
        realReader.initialize(split, taskContext);

        // read once to gain access to key and value objects
        if (realReader.nextKeyValue()) {
          firstRecord = true;
          valueObj = realReader.getCurrentValue();
        } else {
          eof = true;
        }
      } catch (final InterruptedException e) {
        throw new IOException(e);
      }
    } else {
      realReader = null;
      eof = true;
    }
    if (valueObj == null) { // Should initialize the value for createValue
      valueObj = new ArrayWritable(Writable.class, new Writable[schemaSize]);
    }
  }

  @Override
  public void close() throws IOException {
    if (realReader != null) {
      realReader.close();
    }
  }

  @Override
  public Void createKey() {
    return null;
  }

  @Override
  public ArrayWritable createValue() {
    return valueObj;
  }

  @Override
  public long getPos() throws IOException {
    return (long) (splitLen * getProgress());
  }

  @Override
  public float getProgress() throws IOException {
    if (realReader == null) {
      return 1f;
    } else {
      try {
        return realReader.getProgress();
      } catch (final InterruptedException e) {
        throw new IOException(e);
      }
    }
  }

  @Override
  public boolean next(final Void key, final ArrayWritable value) throws IOException {
    if (eof) {
      return false;
    }
    try {
      if (firstRecord) { // key & value are already read.
        firstRecord = false;
      } else if (!realReader.nextKeyValue()) {
        eof = true; // strictly not required, just for consistency
        return false;
      }

      final ArrayWritable tmpCurValue = realReader.getCurrentValue();
      if (value != tmpCurValue) {
        final Writable[] arrValue = value.get();
        final Writable[] arrCurrent = tmpCurValue.get();
        if (value != null && arrValue.length == arrCurrent.length) {
          System.arraycopy(arrCurrent, 0, arrValue, 0, arrCurrent.length);
        } else {
          if (arrValue.length != arrCurrent.length) {
            throw new IOException("DeprecatedParquetHiveInput : size of object differs. Value" +
              " size :  " + arrValue.length + ", Current Object size : " + arrCurrent.length);
          } else {
            throw new IOException("DeprecatedParquetHiveInput can not support RecordReaders that" +
              " don't return same key & value & value is null");
          }
        }
      }
      return true;
    } catch (final InterruptedException e) {
      throw new IOException(e);
    }
  }

  /**
   * gets a ParquetInputSplit corresponding to a split given by Hive
   *
   * @param oldSplit The split given by Hive
   * @param conf The JobConf of the Hive job
   * @return a ParquetInputSplit corresponding to the oldSplit
   * @throws IOException if the config cannot be enhanced or if the footer cannot be read from the file
   */
  protected ParquetInputSplit getSplit(
      final InputSplit oldSplit,
      final JobConf conf
      ) throws IOException {
    if (oldSplit instanceof FileSplit) {
      FileSplit fileSplit = (FileSplit) oldSplit;
      final long splitStart = fileSplit.getStart();
      final long splitLength = fileSplit.getLength();
      final Path finalPath = fileSplit.getPath();
      final JobConf cloneJob = hiveBinding.pushProjectionsAndFilters(conf, finalPath.getParent());

      final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(cloneJob, finalPath, SKIP_ROW_GROUPS);
      final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
      final ReadContext readContext =
          new DataWritableReadSupport()
            .init(cloneJob, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema());

      schemaSize = MessageTypeParser.parseMessageType(
            readContext.getReadSupportMetadata().get(DataWritableReadSupport.HIVE_SCHEMA_KEY)
          ).getFieldCount();
       return new ParquetInputSplit(
                finalPath,
                splitStart,
                splitStart + splitLength,
                splitLength,
                fileSplit.getLocations(),
                null);
    } else {
      throw new IllegalArgumentException("Unknown split type: " + oldSplit);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import static org.apache.parquet.Preconditions.checkNotNull;
import static org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE;
import static org.apache.parquet.hadoop.util.ContextUtil.getConfiguration;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.hadoop.ParquetFileWriter.Mode;
import org.apache.parquet.hadoop.api.WriteSupport;
import org.apache.parquet.hadoop.api.WriteSupport.WriteContext;
import org.apache.parquet.hadoop.codec.CodecConfig;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.util.ConfigurationUtil;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * OutputFormat to write to a Parquet file
 *
 * It requires a {@link WriteSupport} to convert the actual records to the underlying format.
 * It requires the schema of the incoming records. (provided by the write support)
 * It allows storing extra metadata in the footer (for example: for schema compatibility purpose when converting from a different schema language).
 *
 * The format configuration settings in the job configuration:
 *

 * # The block size is the size of a row group being buffered in memory
 * # this limits the memory usage when writing
 * # Larger values will improve the IO when reading but consume more memory when writing
 * parquet.block.size=134217728 # in bytes, default = 128 * 1024 * 1024
 *
 * # The page size is for compression. When reading, each page can be decompressed independently.
 * # A block is composed of pages. The page is the smallest unit that must be read fully to access a single record.
 * # If this value is too small, the compression will deteriorate
 * parquet.page.size=1048576 # in bytes, default = 1 * 1024 * 1024
 *
 * # There is one dictionary page per column per row group when dictionary encoding is used.
 * # The dictionary page size works like the page size but for dictionary
 * parquet.dictionary.page.size=1048576 # in bytes, default = 1 * 1024 * 1024
 *
 * # The compression algorithm used to compress pages
 * parquet.compression=UNCOMPRESSED # one of: UNCOMPRESSED, SNAPPY, GZIP, LZO. Default: UNCOMPRESSED. Supersedes mapred.output.compress*
 *
 * # The write support class to convert the records written to the OutputFormat into the events accepted by the record consumer
 * # Usually provided by a specific ParquetOutputFormat subclass
 * parquet.write.support.class= # fully qualified name
 *
 * # To enable/disable dictionary encoding
 * parquet.enable.dictionary=true # false to disable dictionary encoding
 *
 * # To enable/disable summary metadata aggregation at the end of a MR job
 * # The default is true (enabled)
 * parquet.enable.summary-metadata=true # false to disable summary aggregation
 *
 * # Maximum size (in bytes) allowed as padding to align row groups
 * # This is also the minimum size of a row group. Default: 0
 * parquet.writer.max-padding=2097152 # 2 MB
 *

 *
 * If parquet.compression is not set, the following properties are checked (FileOutputFormat behavior).
 * Note that we explicitely disallow custom Codecs
 *

 * mapred.output.compress=true
 * mapred.output.compression.codec=org.apache.hadoop.io.compress.SomeCodec # the codec must be one of Snappy, GZip or LZO
 *

 *
 * if none of those is set the data is uncompressed.
 *
 * @param <T> the type of the materialized records
 */
public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> {
  private static final Logger LOG = LoggerFactory.getLogger(ParquetOutputFormat.class);

  public static enum JobSummaryLevel {
    /**
     * Write no summary files
     */
    NONE,
    /**
     * Write both summary file with row group info and summary file without
     * (both _metadata and _common_metadata)
     */
    ALL,
    /**
     * Write only the summary file without the row group info
     * (_common_metadata only)
     */
    COMMON_ONLY
  }

  /**
   * An alias for JOB_SUMMARY_LEVEL, where true means ALL and false means NONE
   */
  @Deprecated
  public static final String ENABLE_JOB_SUMMARY   = "parquet.enable.summary-metadata";

  /**
   * Must be one of the values in {@link JobSummaryLevel} (case insensitive)
   */
  public static final String JOB_SUMMARY_LEVEL = "parquet.summary.metadata.level";
  public static final String BLOCK_SIZE           = "parquet.block.size";
  public static final String PAGE_SIZE            = "parquet.page.size";
  public static final String COMPRESSION          = "parquet.compression";
  public static final String WRITE_SUPPORT_CLASS  = "parquet.write.support.class";
  public static final String DICTIONARY_PAGE_SIZE = "parquet.dictionary.page.size";
  public static final String ENABLE_DICTIONARY    = "parquet.enable.dictionary";
  public static final String VALIDATION           = "parquet.validation";
  public static final String WRITER_VERSION       = "parquet.writer.version";
  public static final String MEMORY_POOL_RATIO    = "parquet.memory.pool.ratio";
  public static final String MIN_MEMORY_ALLOCATION = "parquet.memory.min.chunk.size";
  public static final String MAX_PADDING_BYTES    = "parquet.writer.max-padding";
  public static final String MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.min";
  public static final String MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.max";
  public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate";
  public static final String COLUMN_INDEX_TRUNCATE_LENGTH = "parquet.columnindex.truncate.length";
  public static final String PAGE_ROW_COUNT_LIMIT = "parquet.page.row.count.limit";

  public static JobSummaryLevel getJobSummaryLevel(Configuration conf) {
    String level = conf.get(JOB_SUMMARY_LEVEL);
    String deprecatedFlag = conf.get(ENABLE_JOB_SUMMARY);

    if (deprecatedFlag != null) {
      LOG.warn("Setting " + ENABLE_JOB_SUMMARY + " is deprecated, please use " + JOB_SUMMARY_LEVEL);
    }

    if (level != null && deprecatedFlag != null) {
      LOG.warn("Both " + JOB_SUMMARY_LEVEL + " and " + ENABLE_JOB_SUMMARY + " are set! " + ENABLE_JOB_SUMMARY + " will be ignored.");
    }

    if (level != null) {
      return JobSummaryLevel.valueOf(level.toUpperCase());
    }

    if (deprecatedFlag != null) {
      return Boolean.valueOf(deprecatedFlag) ? JobSummaryLevel.ALL : JobSummaryLevel.NONE;
    }

    return JobSummaryLevel.ALL;
  }

  public static void setWriteSupportClass(Job job,  Class<?> writeSupportClass) {
    getConfiguration(job).set(WRITE_SUPPORT_CLASS, writeSupportClass.getName());
  }

  public static void setWriteSupportClass(JobConf job, Class<?> writeSupportClass) {
      job.set(WRITE_SUPPORT_CLASS, writeSupportClass.getName());
  }

  public static Class<?> getWriteSupportClass(Configuration configuration) {
    final String className = configuration.get(WRITE_SUPPORT_CLASS);
    if (className == null) {
      return null;
    }
    final Class<?> writeSupportClass = ConfigurationUtil.getClassFromConfig(configuration, WRITE_SUPPORT_CLASS, WriteSupport.class);
    return writeSupportClass;
  }

  public static void setBlockSize(Job job, int blockSize) {
    getConfiguration(job).setInt(BLOCK_SIZE, blockSize);
  }

  public static void setPageSize(Job job, int pageSize) {
    getConfiguration(job).setInt(PAGE_SIZE, pageSize);
  }

  public static void setDictionaryPageSize(Job job, int pageSize) {
    getConfiguration(job).setInt(DICTIONARY_PAGE_SIZE, pageSize);
  }

  public static void setCompression(Job job, CompressionCodecName compression) {
    getConfiguration(job).set(COMPRESSION, compression.name());
  }

  public static void setEnableDictionary(Job job, boolean enableDictionary) {
    getConfiguration(job).setBoolean(ENABLE_DICTIONARY, enableDictionary);
  }

  public static boolean getEnableDictionary(JobContext jobContext) {
    return getEnableDictionary(getConfiguration(jobContext));
  }

  public static int getBlockSize(JobContext jobContext) {
    return getBlockSize(getConfiguration(jobContext));
  }

  public static int getPageSize(JobContext jobContext) {
    return getPageSize(getConfiguration(jobContext));
  }

  public static int getDictionaryPageSize(JobContext jobContext) {
    return getDictionaryPageSize(getConfiguration(jobContext));
  }

  public static CompressionCodecName getCompression(JobContext jobContext) {
    return getCompression(getConfiguration(jobContext));
  }

  public static boolean isCompressionSet(JobContext jobContext) {
    return isCompressionSet(getConfiguration(jobContext));
  }

  public static void setValidation(JobContext jobContext, boolean validating) {
    setValidation(getConfiguration(jobContext), validating);
  }

  public static boolean getValidation(JobContext jobContext) {
    return getValidation(getConfiguration(jobContext));
  }

  public static boolean getEnableDictionary(Configuration configuration) {
    return configuration.getBoolean(
        ENABLE_DICTIONARY, ParquetProperties.DEFAULT_IS_DICTIONARY_ENABLED);
  }

  public static int getMinRowCountForPageSizeCheck(Configuration configuration) {
    return configuration.getInt(MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK,
        ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK);
  }

  public static int getMaxRowCountForPageSizeCheck(Configuration configuration) {
    return configuration.getInt(MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK,
        ParquetProperties.DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK);
  }

  public static boolean getEstimatePageSizeCheck(Configuration configuration) {
    return configuration.getBoolean(ESTIMATE_PAGE_SIZE_CHECK,
        ParquetProperties.DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK);
  }

  @Deprecated
  public static int getBlockSize(Configuration configuration) {
    return configuration.getInt(BLOCK_SIZE, DEFAULT_BLOCK_SIZE);
  }

  public static long getLongBlockSize(Configuration configuration) {
    return configuration.getLong(BLOCK_SIZE, DEFAULT_BLOCK_SIZE);
  }

  public static int getPageSize(Configuration configuration) {
    return configuration.getInt(PAGE_SIZE, ParquetProperties.DEFAULT_PAGE_SIZE);
  }

  public static int getDictionaryPageSize(Configuration configuration) {
    return configuration.getInt(
        DICTIONARY_PAGE_SIZE, ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE);
  }

  public static WriterVersion getWriterVersion(Configuration configuration) {
    String writerVersion = configuration.get(
        WRITER_VERSION, ParquetProperties.DEFAULT_WRITER_VERSION.toString());
    return WriterVersion.fromString(writerVersion);
  }

  public static CompressionCodecName getCompression(Configuration configuration) {
    return CodecConfig.getParquetCompressionCodec(configuration);
  }

  public static boolean isCompressionSet(Configuration configuration) {
    return CodecConfig.isParquetCompressionSet(configuration);
  }

  public static void setValidation(Configuration configuration, boolean validating) {
    configuration.setBoolean(VALIDATION, validating);
  }

  public static boolean getValidation(Configuration configuration) {
    return configuration.getBoolean(VALIDATION, false);
  }

  private CompressionCodecName getCodec(TaskAttemptContext taskAttemptContext) {
    return CodecConfig.from(taskAttemptContext).getCodec();
  }

  public static void setMaxPaddingSize(JobContext jobContext, int maxPaddingSize) {
    setMaxPaddingSize(getConfiguration(jobContext), maxPaddingSize);
  }

  public static void setMaxPaddingSize(Configuration conf, int maxPaddingSize) {
    conf.setInt(MAX_PADDING_BYTES, maxPaddingSize);
  }

  private static int getMaxPaddingSize(Configuration conf) {
    return conf.getInt(MAX_PADDING_BYTES, ParquetWriter.MAX_PADDING_SIZE_DEFAULT);
  }

  public static void setColumnIndexTruncateLength(JobContext jobContext, int length) {
    setColumnIndexTruncateLength(getConfiguration(jobContext), length);
  }

  public static void setColumnIndexTruncateLength(Configuration conf, int length) {
    conf.setInt(COLUMN_INDEX_TRUNCATE_LENGTH, length);
  }

  private static int getColumnIndexTruncateLength(Configuration conf) {
    return conf.getInt(COLUMN_INDEX_TRUNCATE_LENGTH, ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH);
  }

  public static void setPageRowCountLimit(JobContext jobContext, int rowCount) {
    setPageRowCountLimit(getConfiguration(jobContext), rowCount);
  }

  public static void setPageRowCountLimit(Configuration conf, int rowCount) {
    conf.setInt(PAGE_ROW_COUNT_LIMIT, rowCount);
  }

  private static int getPageRowCountLimit(Configuration conf) {
    return conf.getInt(PAGE_ROW_COUNT_LIMIT, ParquetProperties.DEFAULT_PAGE_ROW_COUNT_LIMIT);
  }

  private WriteSupport<T> writeSupport;
  private ParquetOutputCommitter committer;

  /**
   * constructor used when this OutputFormat in wrapped in another one (In Pig for example)
   *
   * @param writeSupport the class used to convert the incoming records
   * @param <S> the Java write support type
   */
  public <S extends WriteSupport<T>> ParquetOutputFormat(S writeSupport) {
    this.writeSupport = writeSupport;
  }

  /**
   * used when directly using the output format and configuring the write support implementation
   * using parquet.write.support.class
   *
   * @param <S> the Java write support type
   */
  public <S extends WriteSupport<T>> ParquetOutputFormat() {
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {

    final Configuration conf = getConfiguration(taskAttemptContext);

    CompressionCodecName codec = getCodec(taskAttemptContext);
    String extension = codec.getExtension() + ".parquet";
    Path file = getDefaultWorkFile(taskAttemptContext, extension);
    return getRecordWriter(conf, file, codec);
  }

  public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Path file)
      throws IOException, InterruptedException {
    return getRecordWriter(getConfiguration(taskAttemptContext), file, getCodec(taskAttemptContext));
  }

  public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec)
        throws IOException, InterruptedException {
    final WriteSupport<T> writeSupport = getWriteSupport(conf);

    ParquetProperties props = ParquetProperties.builder()
        .withPageSize(getPageSize(conf))
        .withDictionaryPageSize(getDictionaryPageSize(conf))
        .withDictionaryEncoding(getEnableDictionary(conf))
        .withWriterVersion(getWriterVersion(conf))
        .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf))
        .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf))
        .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf))
        .withColumnIndexTruncateLength(getColumnIndexTruncateLength(conf))
        .withPageRowCountLimit(getPageRowCountLimit(conf))
        .build();

    long blockSize = getLongBlockSize(conf);
    int maxPaddingSize = getMaxPaddingSize(conf);
    boolean validating = getValidation(conf);

    if (LOG.isInfoEnabled()) {
      LOG.info("Parquet block size to {}", blockSize);
      LOG.info("Parquet page size to {}", props.getPageSizeThreshold());
      LOG.info("Parquet dictionary page size to {}", props.getDictionaryPageSizeThreshold());
      LOG.info("Dictionary is {}", (props.isEnableDictionary() ? "on" : "off"));
      LOG.info("Validation is {}", (validating ? "on" : "off"));
      LOG.info("Writer version is: {}", props.getWriterVersion());
      LOG.info("Maximum row group padding size is {} bytes", maxPaddingSize);
      LOG.info("Page size checking is: {}", (props.estimateNextSizeCheck() ? "estimated" : "constant"));
      LOG.info("Min row count for page size check is: {}", props.getMinRowCountForPageSizeCheck());
      LOG.info("Max row count for page size check is: {}", props.getMaxRowCountForPageSizeCheck());
      LOG.info("Truncate length for column indexes is: {}", props.getColumnIndexTruncateLength());
      LOG.info("Page row count limit to {}", props.getPageRowCountLimit());
    }

    WriteContext init = writeSupport.init(conf);
    ParquetFileWriter w = new ParquetFileWriter(HadoopOutputFile.fromPath(file, conf),
        init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize, props.getColumnIndexTruncateLength());
    w.start();

    float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,
        MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
    long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION,
        MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION);
    synchronized (ParquetOutputFormat.class) {
      if (memoryManager == null) {
        memoryManager = new MemoryManager(maxLoad, minAllocation);
      }
    }
    if (memoryManager.getMemoryPoolRatio() != maxLoad) {
      LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " +
          "be reset by the new value: " + maxLoad);
    }

    return new ParquetRecordWriter<T>(
        w,
        writeSupport,
        init.getSchema(),
        init.getExtraMetaData(),
        blockSize,
        codec,
        validating,
        props,
        memoryManager,
        conf);
  }

  /**
   * @param configuration to find the configuration for the write support class
   * @return the configured write support
   */
  @SuppressWarnings("unchecked")
  public WriteSupport<T> getWriteSupport(Configuration configuration){
    if (writeSupport != null) return writeSupport;
    Class<?> writeSupportClass = getWriteSupportClass(configuration);
    try {
      return (WriteSupport<T>)checkNotNull(writeSupportClass, "writeSupportClass").newInstance();
    } catch (InstantiationException e) {
      throw new BadConfigurationException("could not instantiate write support class: " + writeSupportClass, e);
    } catch (IllegalAccessException e) {
      throw new BadConfigurationException("could not instantiate write support class: " + writeSupportClass, e);
    }
  }

  @Override
  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException {
    if (committer == null) {
      Path output = getOutputPath(context);
      committer = new ParquetOutputCommitter(output, context);
    }
    return committer;
  }

  /**
   * This memory manager is for all the real writers (InternalParquetRecordWriter) in one task.
   */
  private static MemoryManager memoryManager;

  public synchronized static MemoryManager getMemoryManager() {
    return memoryManager;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.bytes;

import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;

/**
 * Based on DataInputStream but little endian and without the String/char methods
 */
public final class LittleEndianDataInputStream extends InputStream {

  private final InputStream in;

  /**
   * Creates a LittleEndianDataInputStream that uses the specified
   * underlying InputStream.
   *
   * @param  in   the specified input stream
   */
  public LittleEndianDataInputStream(InputStream in) {
    this.in = in;
  }

  /**
   * See the general contract of the readFully
   * method of DataInput.
   *

   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @param      b   the buffer into which the data is read.
   * @exception  EOFException  if this input stream reaches the end before
   *             reading all the bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final void readFully(byte b[]) throws IOException {
    readFully(b, 0, b.length);
  }

  /**
   * See the general contract of the readFully
   * method of DataInput.
   *

   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @param      b     the buffer into which the data is read.
   * @param      off   the start offset of the data.
   * @param      len   the number of bytes to read.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading all the bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final void readFully(byte b[], int off, int len) throws IOException {
    if (len < 0)
      throw new IndexOutOfBoundsException();
    int n = 0;
    while (n < len) {
      int count = in.read(b, off + n, len - n);
      if (count < 0)
        throw new EOFException();
      n += count;
    }
  }

  /**
   * See the general contract of the skipBytes
   * method of DataInput.
   *

   * Bytes for this operation are read from the contained
   * input stream.
   *
   * @param      n   the number of bytes to be skipped.
   * @return     the actual number of bytes skipped.
   * @exception  IOException  if the contained input stream does not support
   *             seek, or the stream has been closed and
   *             the contained input stream does not support
   *             reading after close, or another I/O error occurs.
   */
  public final int skipBytes(int n) throws IOException {
    int total = 0;
    int cur = 0;

    while ((total<n) && ((cur = (int) in.skip(n-total)) > 0)) {
      total += cur;
    }

    return total;
  }

  /**
   * @return the next byte in the stream as an int
   * @throws IOException if there is an exception while reading
   * @see java.io.InputStream#read()
   */
  public int read() throws IOException {
    return in.read();
  }

  /**
   * @return the hash code of the wrapped input stream
   * @see java.lang.Object#hashCode()
   */
  public int hashCode() {
    return in.hashCode();
  }

  /**
   * @param b a byte array
   * @return the number of bytes read
   * @throws IOException if there was an exception while reading
   * @see java.io.InputStream#read(byte[])
   */
  public int read(byte[] b) throws IOException {
    return in.read(b);
  }

  /**
   * @param obj another object
   * @return true if this is equal to the object
   * @see java.lang.Object#equals(java.lang.Object)
   */
  public boolean equals(Object obj) {
    return in.equals(obj);
  }

  /**
   * @param b a byte array
   * @param off an offset into the byte array
   * @param len the length to read
   * @return the number of bytes read
   * @throws IOException if there was an exception while reading
   * @see java.io.InputStream#read(byte[], int, int)
   */
  public int read(byte[] b, int off, int len) throws IOException {
    return in.read(b, off, len);
  }

  /**
   * @param n the number of bytes to skip
   * @return the number of bytes skipped
   * @throws IOException if there was an exception while reading
   * @see java.io.InputStream#skip(long)
   */
  public long skip(long n) throws IOException {
    return in.skip(n);
  }

  /**
   * @return the number of bytes available in the wrapped stream
   * @throws IOException if there was an exception while getting the number of available bytes
   * @see java.io.InputStream#available()
   */
  public int available() throws IOException {
    return in.available();
  }

  /**
   * @throws IOException if there was an exception while closing the underlying stream
   * @see java.io.InputStream#close()
   */
  public void close() throws IOException {
    in.close();
  }

  /**
   * @param readlimit the number of bytes the mark will be valid for
   * @see java.io.InputStream#mark(int)
   */
  public void mark(int readlimit) {
    in.mark(readlimit);
  }

  /**
   * @throws IOException if there is an exception while resetting the underlying stream
   * @see java.io.InputStream#reset()
   */
  public void reset() throws IOException {
    in.reset();
  }

  /**
   * @return true if mark is supported
   * @see java.io.InputStream#markSupported()
   */
  public boolean markSupported() {
    return in.markSupported();
  }

  /**
   * See the general contract of the readBoolean
   * method of DataInput.
   *

   * Bytes for this operation are read from the contained
   * input stream.
   *
   * @return     the boolean value read.
   * @exception  EOFException  if this input stream has reached the end.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final boolean readBoolean() throws IOException {
    int ch = in.read();
    if (ch < 0)
      throw new EOFException();
    return (ch != 0);
  }

  /**
   * See the general contract of the readByte
   * method of DataInput.
   *

   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next byte of this input stream as a signed 8-bit
   *             byte.
   * @exception  EOFException  if this input stream has reached the end.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final byte readByte() throws IOException {
    int ch = in.read();
    if (ch < 0)
      throw new EOFException();
    return (byte)(ch);
  }

  /**
   * See the general contract of the readUnsignedByte
   * method of DataInput.
   *

   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next byte of this input stream, interpreted as an
   *             unsigned 8-bit number.
   * @exception  EOFException  if this input stream has reached the end.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see         java.io.FilterInputStream#in
   */
  public final int readUnsignedByte() throws IOException {
    int ch = in.read();
    if (ch < 0)
      throw new EOFException();
    return ch;
  }

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next two bytes of this input stream, interpreted as a
   *             signed 16-bit number.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading two bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final short readShort() throws IOException {
    int ch2 = in.read();
    int ch1 = in.read();
    if ((ch1 | ch2) < 0)
      throw new EOFException();
    return (short)((ch1 << 8) + (ch2 << 0));
  }

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next two bytes of this input stream, interpreted as an
   *             unsigned 16-bit integer.
   * @exception  EOFException  if this input stream reaches the end before
   *             reading two bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final int readUnsignedShort() throws IOException {
    int ch2 = in.read();
    int ch1 = in.read();
    if ((ch1 | ch2) < 0)
      throw new EOFException();
    return (ch1 << 8) + (ch2 << 0);
  }

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next four bytes of this input stream, interpreted as an
   *             int.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading four bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final int readInt() throws IOException {
    // TODO: has this been benchmarked against two alternate implementations?
    // 1) Integer.reverseBytes(in.readInt())
    // 2) keep a member byte[4], wrapped by an IntBuffer with appropriate endianness set,
    //    and call IntBuffer.get()
    // Both seem like they might be faster.
    int ch4 = in.read();
    int ch3 = in.read();
    int ch2 = in.read();
    int ch1 = in.read();
    if ((ch1 | ch2 | ch3 | ch4) < 0)
      throw new EOFException();
    return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
  }

  private byte readBuffer[] = new byte[8];

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next eight bytes of this input stream, interpreted as a
   *             long.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading eight bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.io.FilterInputStream#in
   */
  public final long readLong() throws IOException {
    // TODO: see perf question above in readInt
    readFully(readBuffer, 0, 8);
    return (((long)readBuffer[7] << 56) +
        ((long)(readBuffer[6] & 255) << 48) +
        ((long)(readBuffer[5] & 255) << 40) +
        ((long)(readBuffer[4] & 255) << 32) +
        ((long)(readBuffer[3] & 255) << 24) +
        ((readBuffer[2] & 255) << 16) +
        ((readBuffer[1] & 255) <<  8) +
        ((readBuffer[0] & 255) <<  0));
  }

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next four bytes of this input stream, interpreted as a
   *             float.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading four bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.lang.Float#intBitsToFloat(int)
   */
  public final float readFloat() throws IOException {
    return Float.intBitsToFloat(readInt());
  }

  /**
   * Bytes
   * for this operation are read from the contained
   * input stream.
   *
   * @return     the next eight bytes of this input stream, interpreted as a
   *             double.
   * @exception  EOFException  if this input stream reaches the end before
   *               reading eight bytes.
   * @exception  IOException   the stream has been closed and the contained
   *             input stream does not support reading after close, or
   *             another I/O error occurs.
   * @see        java.lang.Double#longBitsToDouble(long)
   */
  public final double readDouble() throws IOException {
    return Double.longBitsToDouble(readLong());
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;

/**
 * An input split for the Parquet format
 * It contains the information to read one block of the file.
 *
 * This class is private to the ParquetInputFormat.
 * Backward compatibility is not maintained.
 * @deprecated will be removed in 2.0.0. use FileInputSplit instead.
 */
@Private
@Deprecated
public class ParquetInputSplit extends FileSplit implements Writable {

  private long end;
  private long[] rowGroupOffsets;

  /**
   * Writables must have a parameterless constructor
   */
  public ParquetInputSplit() {
        super(null, 0, 0, new String[0]);
  }

  /**
   * For compatibility only
   * use {@link ParquetInputSplit#ParquetInputSplit(Path, long, long, long, String[], long[])}
   * @param path a Path
   * @param start split start location
   * @param length split length
   * @param hosts locality information for this split
   * @param blocks Parquet blocks in this split
   * @param requestedSchema the requested schema
   * @param fileSchema the file schema
   * @param extraMetadata string map of file metadata
   * @param readSupportMetadata string map of metadata from read support
   */
  @Deprecated
  public ParquetInputSplit(
      Path path,
      long start,
      long length,
      String[] hosts,
      List<BlockMetaData> blocks,
      String requestedSchema,
      String fileSchema,
      Map<String, String> extraMetadata,
      Map<String, String> readSupportMetadata) {
    this(path, start, end(blocks, requestedSchema), length, hosts, offsets(blocks));
  }

  private static long end(List<BlockMetaData> blocks, String requestedSchema) {
    MessageType requested = MessageTypeParser.parseMessageType(requestedSchema);
    long length = 0;

    for (BlockMetaData block : blocks) {
      List<ColumnChunkMetaData> columns = block.getColumns();
      for (ColumnChunkMetaData column : columns) {
        if (requested.containsPath(column.getPath().toArray())) {
          length += column.getTotalSize();
        }
      }
    }
    return length;
  }

  private static long[] offsets(List<BlockMetaData> blocks) {
    long[] offsets = new long[blocks.size()];
    for (int i = 0; i < offsets.length; i++) {
      offsets[i] = blocks.get(i).getStartingPos();
    }
    return offsets;
  }

  /**
   * @return the block meta data
   * @deprecated the file footer is no longer read before creating input splits
   */
  @Deprecated
  public List<BlockMetaData> getBlocks() {
    throw new UnsupportedOperationException(
        "Splits no longer have row group metadata, see PARQUET-234");
  }

  /**
   * Builds a {@code ParquetInputSplit} from a mapreduce {@link FileSplit}.
   *
   * @param split a mapreduce FileSplit
   * @return a ParquetInputSplit
   * @throws IOException if there is an error while creating the Parquet split
   */
  static ParquetInputSplit from(FileSplit split) throws IOException {
    return new ParquetInputSplit(split.getPath(),
        split.getStart(), split.getStart() + split.getLength(),
        split.getLength(), split.getLocations(), null);
  }

  /**
   * Builds a {@code ParquetInputSplit} from a mapred
   * {@link org.apache.hadoop.mapred.FileSplit}.
   *
   * @param split a mapred FileSplit
   * @return a ParquetInputSplit
   * @throws IOException if there is an error while creating the Parquet split
   */
  static ParquetInputSplit from(org.apache.hadoop.mapred.FileSplit split) throws IOException {
    return new ParquetInputSplit(split.getPath(),
        split.getStart(), split.getStart() + split.getLength(),
        split.getLength(), split.getLocations(), null);
  }

  /**
   * @param file the path of the file for that split
   * @param start the start offset in the file
   * @param end the end offset in the file
   * @param length the actual size in bytes that we expect to read
   * @param hosts the hosts with the replicas of this data
   * @param rowGroupOffsets the offsets of the rowgroups selected if loaded on the client
   */
  public ParquetInputSplit(
      Path file, long start, long end, long length, String[] hosts,
      long[] rowGroupOffsets) {
    super(file, start, length, hosts);
    this.end = end;
    this.rowGroupOffsets = rowGroupOffsets;
  }

  /**
   * @return the requested schema
   * @deprecated the file footer is no longer read before creating input splits
   */
  @Deprecated
  String getRequestedSchema() {
    throw new UnsupportedOperationException(
        "Splits no longer have the requested schema, see PARQUET-234");
  }

  /**
   * @return the file schema
   * @deprecated the file footer is no longer read before creating input splits
   */
  @Deprecated
  public String getFileSchema() {
    throw new UnsupportedOperationException(
        "Splits no longer have the file schema, see PARQUET-234");
  }

  /**
   * @return the end offset of that split
   */
  public long getEnd() {
    return end;
  }

  /**
   * @return app specific metadata from the file
   * @deprecated will be removed in 2.0.0. the file footer is no longer read before creating input splits
   */
  @Deprecated
  public Map<String, String> getExtraMetadata() {
    throw new UnsupportedOperationException(
        "Splits no longer have file metadata, see PARQUET-234");
  }

  /**
   * @return app specific metadata provided by the read support in the init phase
   * @deprecated will be removed in 2.0.0.
   */
  @Deprecated
  Map<String, String> getReadSupportMetadata() {
    throw new UnsupportedOperationException(
        "Splits no longer have read-support metadata, see PARQUET-234");
  }

  /**
   * @return the offsets of the row group selected if this has been determined on the client side
   * @deprecated will be removed in 2.0.0.
   */
  public long[] getRowGroupOffsets() {
    return rowGroupOffsets;
  }

  @Override
  public String toString() {
    String hosts;
    try{
       hosts = Arrays.toString(getLocations());
    } catch (Exception e) {
      // IOException/InterruptedException could be thrown
      hosts = "(" + e + ")";
    }

    return this.getClass().getSimpleName() + "{" +
           "part: " + getPath()
        + " start: " + getStart()
        + " end: " + getEnd()
        + " length: " + getLength()
        + " hosts: " + hosts
        + (rowGroupOffsets == null ? "" : (" row groups: " + Arrays.toString(rowGroupOffsets)))
        + "}";
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void readFields(DataInput hin) throws IOException {
    byte[] bytes = readArray(hin);
    DataInputStream in = new DataInputStream(new GZIPInputStream(new ByteArrayInputStream(bytes)));
    super.readFields(in);
    this.end = in.readLong();
    if (in.readBoolean()) {
      this.rowGroupOffsets = new long[in.readInt()];
      for (int i = 0; i < rowGroupOffsets.length; i++) {
        rowGroupOffsets[i] = in.readLong();
      }
    }
    in.close();
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void write(DataOutput hout) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(new GZIPOutputStream(baos));
    super.write(out);
    out.writeLong(end);
    out.writeBoolean(rowGroupOffsets != null);
    if (rowGroupOffsets != null) {
      out.writeInt(rowGroupOffsets.length);
      for (long o : rowGroupOffsets) {
        out.writeLong(o);
      }
    }
    out.close();
    writeArray(hout, baos.toByteArray());
  }

  private static void writeArray(DataOutput out, byte[] bytes) throws IOException {
    out.writeInt(bytes.length);
    out.write(bytes, 0, bytes.length);
  }

  private static byte[] readArray(DataInput in) throws IOException {
    int len = in.readInt();
    byte[] bytes = new byte[len];
    in.readFully(bytes);
    return bytes;
  }

}