/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift.pig;

import java.io.IOException;
import java.util.Arrays;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.pig.StoreFunc;
import org.apache.pig.data.Tuple;

import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.io.ParquetEncodingException;

/**
 * To store in Pig using a thrift class
 * usage:
 * STORE 'foo' USING parquet.thrift.pig.ParquetThriftStorer('my.thrift.Class');
 */
public class ParquetThriftStorer extends StoreFunc {

  private RecordWriter<Void, Tuple> recordWriter;

  private String className;

  public ParquetThriftStorer(String[] params) {
    if (params == null || params.length != 1) {
      throw new IllegalArgumentException("required the thrift class name in parameter. Got " + Arrays.toString(params) + " instead");
    }
    className = params[0];
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public OutputFormat<Void, Tuple> getOutputFormat() throws IOException {
    return new ParquetOutputFormat<Tuple>(new TupleToThriftWriteSupport(className));
  }

  /**
   * {@inheritDoc}
   */
  @SuppressWarnings({ "rawtypes", "unchecked" }) // that's how the base class is defined
  @Override
  public void prepareToWrite(RecordWriter recordWriter) throws IOException {
    this.recordWriter = recordWriter;
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void putNext(Tuple tuple) throws IOException {
    try {
      this.recordWriter.write(null, tuple);
    } catch (InterruptedException e) {
      throw new ParquetEncodingException("Interrupted while writing", e);
    }
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void setStoreLocation(String location, Job job) throws IOException {
    FileOutputFormat.setOutputPath(job, new Path(location));
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.delta;

import java.io.IOException;

import org.apache.parquet.Preconditions;
import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
import org.apache.parquet.column.values.bitpacking.Packer;
import org.apache.parquet.io.ParquetEncodingException;

/**
 * Write longs (INT64) with delta encoding and binary packing.
 */
public class DeltaBinaryPackingValuesWriterForLong extends DeltaBinaryPackingValuesWriter {
  /**
   * max bitwidth for a mini block, it is used to allocate miniBlockByteBuffer which is
   * reused between flushes.
   */
  private static final int MAX_BITWIDTH = 64;

  /**
   * stores delta values starting from the 2nd value written(1st value is stored in header).
   * It's reused between flushes
   */
  private long[] deltaBlockBuffer;

  /**
   * firstValue is written to the header of the page
   */
  private long firstValue = 0;

  /**
   * cache previous written value for calculating delta
   */
  private long previousValue = 0;

  /**
   * min delta is written to the beginning of each block.
   * it's zig-zag encoded. The deltas stored in each block is actually the difference to min delta,
   * therefore are all positive
   * it will be reset after each flush
   */
  private long minDeltaInCurrentBlock = Long.MAX_VALUE;

  public DeltaBinaryPackingValuesWriterForLong(
      int slabSize, int pageSize, ByteBufferAllocator allocator) {
    this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, slabSize, pageSize, allocator);
  }

  public DeltaBinaryPackingValuesWriterForLong(int blockSizeInValues, int miniBlockNum,
      int slabSize, int pageSize, ByteBufferAllocator allocator) {
    super(blockSizeInValues, miniBlockNum, slabSize, pageSize, allocator);
    deltaBlockBuffer = new long[config.blockSizeInValues];
    miniBlockByteBuffer = new byte[config.miniBlockSizeInValues * MAX_BITWIDTH];
  }

  @Override
  public void writeLong(long v) {
    totalValueCount++;

    if (totalValueCount == 1) {
      firstValue = v;
      previousValue = firstValue;
      return;
    }

    // Calculate delta. The possible overflow is accounted for. The algorithm is correct because
    // Java long is working as a modalar ring with base 2^64 and because of the plus and minus
    // properties of a ring. http://en.wikipedia.org/wiki/Modular_arithmetic#Integers_modulo_n
    long delta = v - previousValue;
    previousValue = v;

    deltaBlockBuffer[deltaValuesToFlush++] = delta;

    if (delta < minDeltaInCurrentBlock) {
      minDeltaInCurrentBlock = delta;
    }

    if (config.blockSizeInValues == deltaValuesToFlush) {
      flushBlockBuffer();
    }
  }

  private void flushBlockBuffer() {
    // since we store the min delta, the deltas will be converted to be the difference to min delta
    // and all positive
    for (int i = 0; i < deltaValuesToFlush; i++) {
      deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaInCurrentBlock;
    }

    writeMinDelta();
    int miniBlocksToFlush = getMiniBlockCountToFlush(deltaValuesToFlush);

    calculateBitWidthsForDeltaBlockBuffer(miniBlocksToFlush);
    for (int i = 0; i < config.miniBlockNumInABlock; i++) {
      writeBitWidthForMiniBlock(i);
    }

    for (int i = 0; i < miniBlocksToFlush; i++) {
      // writing i th miniblock
      int currentBitWidth = bitWidths[i];
      int blockOffset = 0;
      // TODO: should this cache the packer?
      BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(currentBitWidth);
      int miniBlockStart = i * config.miniBlockSizeInValues;
      // pack values into the miniblock buffer, 8 at a time to get exactly currentBitWidth bytes
      for (int j = miniBlockStart; j < (i + 1) * config.miniBlockSizeInValues; j += 8) {
        // mini block is atomic in terms of flushing
        // This may write more values when reach to the end of data writing to last mini block,
        // since it may not be aligned to miniblock,
        // but doesn't matter. The reader uses total count to see if reached the end.
        packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset);
        blockOffset += currentBitWidth;
      }
      baos.write(miniBlockByteBuffer, 0, blockOffset);
    }

    minDeltaInCurrentBlock = Long.MAX_VALUE;
    deltaValuesToFlush = 0;
  }

  private void writeMinDelta() {
    try {
      BytesUtils.writeZigZagVarLong(minDeltaInCurrentBlock, baos);
    } catch (IOException e) {
      throw new ParquetEncodingException("can not write min delta for block", e);
    }
  }

  /**
   * iterate through values in each mini block and calculate the bitWidths of max values.
   *
   * @param miniBlocksToFlush number of miniblocks
   */
  private void calculateBitWidthsForDeltaBlockBuffer(int miniBlocksToFlush) {
    for (int miniBlockIndex = 0; miniBlockIndex < miniBlocksToFlush; miniBlockIndex++) {
      long mask = 0;
      int miniStart = miniBlockIndex * config.miniBlockSizeInValues;

      //The end of current mini block could be the end of current block(deltaValuesToFlush) buffer when data is not aligned to mini block
      int miniEnd = Math.min((miniBlockIndex + 1) * config.miniBlockSizeInValues, deltaValuesToFlush);

      for (int i = miniStart; i < miniEnd; i++) {
        mask |= deltaBlockBuffer[i];
      }
      bitWidths[miniBlockIndex] = 64 - Long.numberOfLeadingZeros(mask);
    }
  }

  /**
   * getBytes will trigger flushing block buffer, DO NOT write after getBytes() is called without calling reset()
   *
   * @return a BytesInput that contains the encoded page data
   */
  @Override
  public BytesInput getBytes() {
    // The Page Header should include: blockSizeInValues, numberOfMiniBlocks, totalValueCount
    if (deltaValuesToFlush != 0) {
      flushBlockBuffer();
    }
    return BytesInput.concat(
            config.toBytesInput(),
            BytesInput.fromUnsignedVarInt(totalValueCount),
            BytesInput.fromZigZagVarLong(firstValue),
            BytesInput.from(baos));
  }

  @Override
  public void reset() {
    super.reset();
    this.minDeltaInCurrentBlock = Long.MAX_VALUE;
  }

  @Override
  public void close() {
    super.close();
    this.minDeltaInCurrentBlock = Long.MAX_VALUE;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet;

/**
 * Utility for parameter validation
 */
public final class Preconditions {
  private Preconditions() { }

  /**
   * @param o the param to check
   * @param name the name of the param for the error message
   * @param <T> the type of the object
   * @return the validated o
   * @throws NullPointerException if o is null
   */
  public static <T> T checkNotNull(T o, String name) throws NullPointerException {
    if (o == null) {
      throw new NullPointerException(name + " should not be null");
    }
    return o;
  }

  /**
   * Precondition-style validation that throws {@link IllegalArgumentException}.
   *
   * @param isValid
   *          {@code true} if valid, {@code false} if an exception should be
   *          thrown
   * @param message
   *          A String message for the exception.
   * @throws IllegalArgumentException if {@code isValid} is false
   */
  public static void checkArgument(boolean isValid, String message) throws IllegalArgumentException {
    if (!isValid) {
      throw new IllegalArgumentException(message);
    }
  }

  /**
   * Precondition-style validation that throws {@link IllegalArgumentException}.
   *
   * @param isValid
   *          {@code true} if valid, {@code false} if an exception should be
   *          thrown
   * @param message
   *          A String message for the exception.
   * @param args
   *          Objects used to fill in {@code %s} placeholders in the message
   * @throws IllegalArgumentException if {@code isValid} is false
   */
  public static void checkArgument(boolean isValid, String message, Object... args)
      throws IllegalArgumentException {
    if (!isValid) {
      throw new IllegalArgumentException(
          String.format(String.valueOf(message), strings(args)));
    }
  }

  /**
   * Precondition-style validation that throws {@link IllegalStateException}.
   *
   * @param isValid
   *          {@code true} if valid, {@code false} if an exception should be
   *          thrown
   * @param message
   *          A String message for the exception.
   * @throws IllegalStateException if {@code isValid} is false
   */
  public static void checkState(boolean isValid, String message) throws IllegalStateException {
    if (!isValid) {
      throw new IllegalStateException(message);
    }
  }

  /**
   * Precondition-style validation that throws {@link IllegalStateException}.
   *
   * @param isValid
   *          {@code true} if valid, {@code false} if an exception should be
   *          thrown
   * @param message
   *          A String message for the exception.
   * @param args
   *          Objects used to fill in {@code %s} placeholders in the message
   * @throws IllegalStateException if {@code isValid} is false
   */
  public static void checkState(boolean isValid, String message, Object... args)
      throws IllegalStateException {
    if (!isValid) {
      throw new IllegalStateException(
          String.format(String.valueOf(message), strings(args)));
    }
  }

  private static String[] strings(Object[] objects) {
    String[] strings = new String[objects.length];
    for (int i = 0; i < objects.length; i += 1) {
      strings[i] = String.valueOf(objects[i]);
    }
    return strings;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift;

import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TMessage;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TStruct;
import org.apache.thrift.protocol.TType;

import org.apache.parquet.ParquetRuntimeException;
import org.apache.parquet.ShouldNeverHappenException;
import org.apache.parquet.thrift.struct.ThriftField;
import org.apache.parquet.thrift.struct.ThriftType;
import org.apache.parquet.thrift.struct.ThriftType.ListType;
import org.apache.parquet.thrift.struct.ThriftType.MapType;
import org.apache.parquet.thrift.struct.ThriftType.SetType;
import org.apache.parquet.thrift.struct.ThriftType.StructType;
import org.apache.parquet.thrift.struct.ThriftType.StructType.StructOrUnionType;
import org.apache.parquet.thrift.struct.ThriftTypeID;

/**
 * Class to read from one protocol in a buffer and then write to another one
 * When there is an exception during reading, it's a skippable exception.
 * When schema is not compatible, the {@link SkippableException} will be thrown.
 * <p>
 * When there are fields in the data that are not defined in the schema, the fields will be ignored and the handler will
 * be notified through {@link FieldIgnoredHandler#handleFieldIgnored(org.apache.thrift.protocol.TField)}
 * and {@link FieldIgnoredHandler#handleRecordHasFieldIgnored()}
 */
public class BufferedProtocolReadToWrite implements ProtocolPipe {

  private interface Action {
    void write(TProtocol out) throws TException;
    String toDebugString();
  }

  private static final Action STRUCT_END = new Action() {
    @Override
    public void write(TProtocol out) throws TException {
      out.writeFieldStop();
      out.writeStructEnd();
    }

    @Override
    public String toDebugString() {
      return ")";
    }
  };

  private static final Action FIELD_END = new Action() {
    @Override
    public void write(TProtocol out) throws TException {
      out.writeFieldEnd();
    }
    @Override
    public String toDebugString() {
      return ";";
    }
  };

  private static final Action MAP_END = new Action() {
    @Override
    public void write(TProtocol out) throws TException {
      out.writeMapEnd();
    }
    @Override
    public String toDebugString() {
      return "]";
    }
  };

  private static final Action LIST_END = new Action() {
    @Override
    public void write(TProtocol out) throws TException {
      out.writeListEnd();
    }
    @Override
    public String toDebugString() {
      return "}";
    }
  };

  private static final Action SET_END = new Action() {
    @Override
    public void write(TProtocol out) throws TException {
      out.writeSetEnd();
    }
    @Override
    public String toDebugString() {
      return "*}";
    }
  };
  //error handler is global
  private final FieldIgnoredHandler errorHandler;
  private final StructType thriftType;

  public BufferedProtocolReadToWrite(StructType thriftType) {
    this(thriftType, null);
  }

  public BufferedProtocolReadToWrite(StructType thriftType, FieldIgnoredHandler errorHandler) {
    super();
    this.thriftType = thriftType;
    this.errorHandler = errorHandler;
  }

  /**
   * Reads one record from in and writes it to out.
   * Exceptions encountered during reading are treated as skippable exceptions,
   * {@link FieldIgnoredHandler} will be notified when registered.
   *
   * @param in  input protocol
   * @param out output protocol
   * @throws org.apache.thrift.TException         when an error happened while writing. Those are usually not recoverable
   */
  @Override
  public void readOne(TProtocol in, TProtocol out) throws TException {
    List<Action> buffer = new LinkedList<Action>();
    try{
        boolean hasFieldsIgnored = readOneStruct(in, buffer, thriftType);
        if (hasFieldsIgnored) {
          notifyRecordHasFieldIgnored();
        }
    } catch (Exception e) {
      throw new SkippableException(error("Error while reading", buffer), e);
    }

    try {
      for (Action a : buffer) {
        a.write(out);
      }
    } catch (Exception e) {
      throw new TException(error("Can not write record", buffer), e);
    }
  }

  private void notifyRecordHasFieldIgnored() {
    if (errorHandler != null) {
      errorHandler.handleRecordHasFieldIgnored();
    }
  }

  private void notifyIgnoredFieldsOfRecord(TField field) {
    if (errorHandler != null) {
      errorHandler.handleFieldIgnored(field);
    }
  }

  private String error(String message, List<Action> buffer) {
    StringBuilder sb = new StringBuilder(message).append(": ");
    for (Action action : buffer) {
      sb.append(action.toDebugString());
    }
    return sb.toString();
  }

  /**
   * @return true when all value is consumed, false when some values is ignored due to the field is not defined in expectedType
   * @throws TException
   */
  private boolean readOneValue(TProtocol in, byte type, List<Action> buffer, ThriftType expectedType) throws TException {
    if (expectedType != null && expectedType.getType().getSerializedThriftType() != type) {
      throw new DecodingSchemaMismatchException("the data type does not match the expected thrift structure: expected " + expectedType + " got " + typeName(type));
    }
    boolean hasFieldsIgnored = false;
    switch (type) {
    case TType.LIST:
      hasFieldsIgnored = readOneList(in, buffer, (ListType)expectedType);
      break;
    case TType.MAP:
      hasFieldsIgnored = readOneMap(in, buffer, (MapType)expectedType);
      break;
    case TType.SET:
      hasFieldsIgnored = readOneSet(in, buffer, (SetType)expectedType);
      break;
    case TType.STRUCT:
      hasFieldsIgnored = readOneStruct(in, buffer, (StructType)expectedType);
      break;
    case TType.STOP:
      break;
    case TType.BOOL:
      final boolean bool = in.readBool();
      writeBoolAction(buffer, bool);
      break;
    case TType.BYTE:
      final byte b = in.readByte();
      writeByteAction(buffer, b);
      break;
    case TType.DOUBLE:
      final double d = in.readDouble();
      writeDoubleAction(buffer, d);
      break;
    case TType.I16:
      final short s = in.readI16();
      writeShortAction(buffer, s);
      break;
    case TType.ENUM: // same as i32 => actually never seen in the protocol layer as enums are written as a i32 field
    case TType.I32:
      final int i = in.readI32();
      checkEnum(expectedType,i);
      writeIntAction(buffer, i);
      break;
    case TType.I64:
      final long l = in.readI64();
      writeLongAction(buffer, l);
      break;
    case TType.STRING:
      final ByteBuffer bin = in.readBinary();
      writeStringAction(buffer, bin);
      break;
    case TType.VOID:
      break;
    default:
      throw new TException("Unknown type: " + type);
    }
    return hasFieldsIgnored;
  }

  private void writeStringAction(List<Action> buffer, final ByteBuffer bin) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeBinary(bin);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(bin);
      }
    });
  }

  private void writeLongAction(List<Action> buffer, final long l) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeI64(l);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(l);
      }
    });
  }

  private void writeIntAction(List<Action> buffer, final int i) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeI32(i);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(i);
      }
    });
  }

  private void writeShortAction(List<Action> buffer, final short s) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeI16(s);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(s);
      }
    });
  }

  private void writeDoubleAction(List<Action> buffer, final double d) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeDouble(d);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(d);
      }
    });
  }

  private void writeByteAction(List<Action> buffer, final byte b) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeByte(b);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(b);
      }
    });
  }

  private void writeBoolAction(List<Action> buffer, final boolean bool) {
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeBool(bool);
      }

      @Override
      public String toDebugString() {
        return String.valueOf(bool);
      }
    });
  }

  private String typeName(byte type) {
    try {
      return ThriftTypeID.fromByte(type).name();
    } catch (RuntimeException e) {
      return String.valueOf(type);
    }
  }

  private boolean readOneStruct(TProtocol in, List<Action> buffer, StructType type) throws TException {
    final TStruct struct = in.readStructBegin();
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeStructBegin(struct);
      }

      @Override
      public String toDebugString() {
        return "(";
      }
    });

    TField field;
    boolean hasFieldsIgnored = false;
    int childFieldsPresent = 0;
    while ((field = in.readFieldBegin()).type != TType.STOP) {
      final TField currentField = field;
      ThriftField expectedField;
      if ((expectedField = type.getChildById(field.id)) == null) {
        handleUnrecognizedField(field, type, in);
        hasFieldsIgnored |= true;
        continue;
      }

      childFieldsPresent++;

      buffer.add(new Action() {
        @Override
        public void write(TProtocol out) throws TException {
          out.writeFieldBegin(currentField);
        }

        @Override
        public String toDebugString() {
          return "f=" + currentField.id + "<t=" + typeName(currentField.type) + ">: ";
        }
      });
      hasFieldsIgnored |= readOneValue(in, field.type, buffer, expectedField.getType());
      in.readFieldEnd();
      buffer.add(FIELD_END);
    }

    // check that union had exactly 1 (no more no less) child fields.
    assertUnionHasExactlyOneChild(type, childFieldsPresent);

    in.readStructEnd();
    buffer.add(STRUCT_END);
    return hasFieldsIgnored;
  }

  private void handleUnrecognizedField(TField field, StructType type, TProtocol in) throws TException {
    switch (type.getStructOrUnionType()) {
      case STRUCT:
        // this is an unrecognized field in a struct, not a union
        notifyIgnoredFieldsOfRecord(field);
        //read the value and ignore it, NullProtocol will do nothing
        new ProtocolReadToWrite().readOneValue(in, new NullProtocol(), field.type);
        break;
      case UNION:
        // this is a union with an unrecognized member -- this is fatal for this record
        // in the write path, because it will be unreadable in the read path.
        // throwing here means we will either skip this record entirely, or fail completely.
        throw new DecodingSchemaMismatchException("Unrecognized union member with id: "
            + field.id + " for struct:\n" + type);
      case UNKNOWN:
        throw unknownStructOrUnion(type);
      default:
        throw unrecognizedStructOrUnion(type.getStructOrUnionType());
    }
  }

  private void assertUnionHasExactlyOneChild(StructType type, int childFieldsPresent) {
    switch (type.getStructOrUnionType()) {
      case STRUCT:
        // nothing to do
        break;
      case UNION:
        // childFieldsPresent must == 1
        if (childFieldsPresent != 1) {

          if (childFieldsPresent == 0) {
            throw new DecodingSchemaMismatchException("Cannot write a TUnion with no set value in :\n" + type);
          } else {
            throw new DecodingSchemaMismatchException("Cannot write a TUnion with more than 1 set value in :\n" + type);
          }

        }
        break;
      case UNKNOWN:
        throw unknownStructOrUnion(type);
      default:
        throw unrecognizedStructOrUnion(type.getStructOrUnionType());
    }
  }

  private static ShouldNeverHappenException unrecognizedStructOrUnion(StructOrUnionType type) {
    return new ShouldNeverHappenException("Unrecognized StructOrUnionType: " + type);
  }

  // we should never reach here in the write path -- this only happens if the
  // deprecated constructor of StructType is used, which should only be used in the
  // read path.
  private static ShouldNeverHappenException unknownStructOrUnion(StructType type) {
    return new ShouldNeverHappenException("This should never happen! "
        + "Don't know if this field is a union, was the deprecated constructor of StructType used?\n" + type);
  }

  private boolean readOneMap(TProtocol in, List<Action> buffer, MapType mapType) throws TException {
    final TMap map = in.readMapBegin();
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeMapBegin(map);
      }

      @Override
      public String toDebugString() {
        return "<k=" + map.keyType + ", v=" + map.valueType + ", s=" + map.size + ">[";
      }
    });
    boolean hasFieldIgnored = false;
    for (int i = 0; i < map.size; i++) {
      hasFieldIgnored |= readOneValue(in, map.keyType, buffer, mapType.getKey().getType());
      hasFieldIgnored |= readOneValue(in, map.valueType, buffer, mapType.getValue().getType());
    }
    in.readMapEnd();
    buffer.add(MAP_END);
    return hasFieldIgnored;
  }

  private boolean readOneSet(TProtocol in, List<Action> buffer, SetType expectedType) throws TException {
    final TSet set = in.readSetBegin();
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeSetBegin(set);
      }

      @Override
      public String toDebugString() {
        return "<e=" + set.elemType + ", s=" + set.size + ">{*";
      }
    });

    boolean hasFieldsIgnored = readCollectionElements(in, set.size, set.elemType, buffer, expectedType.getValues().getType());
    in.readSetEnd();
    buffer.add(SET_END);
    return hasFieldsIgnored;
  }

  private boolean readOneList(TProtocol in, List<Action> buffer, ListType expectedType) throws TException {
    final TList list = in.readListBegin();
    buffer.add(new Action() {
      @Override
      public void write(TProtocol out) throws TException {
        out.writeListBegin(list);
      }

      @Override
      public String toDebugString() {
        return "<e=" + list.elemType + ", s=" + list.size + ">{";
      }
    });
    boolean hasFieldsIgnored = readCollectionElements(in, list.size, list.elemType, buffer, expectedType.getValues().getType());
    in.readListEnd();
    buffer.add(LIST_END);
    return hasFieldsIgnored;
  }

  private boolean readCollectionElements(TProtocol in,
                                         final int size, final byte elemType, List<Action> buffer, ThriftType expectedType) throws TException {
    boolean hasFieldIgnored = false;
    for (int i = 0; i < size; i++) {
      hasFieldIgnored |= readOneValue(in, elemType, buffer, expectedType);
    }
    return hasFieldIgnored;
  }

  /**
   * In thrift enum values are written as ints, this method checks if the enum index is defined.
   *
   * @param expectedType
   * @param i
   */
  private void checkEnum(ThriftType expectedType, int i) {
    if (expectedType.getType() == ThriftTypeID.ENUM) {
      ThriftType.EnumType expectedEnumType = (ThriftType.EnumType)expectedType;
      if (expectedEnumType.getEnumValueById(i) == null) {
        throw new DecodingSchemaMismatchException("can not find index " + i + " in enum " + expectedType);
      }
    }
  }

  /**
   * NullProtocol does nothing when writing to it, used for ignoring unrecognized fields.
   */
  class NullProtocol extends TProtocol {

    public NullProtocol() {
      super(null);
    }

    @Override
    public void writeMessageBegin(TMessage tMessage) throws TException {
    }

    @Override
    public void writeMessageEnd() throws TException {
    }

    @Override
    public void writeStructBegin(TStruct tStruct) throws TException {
    }

    @Override
    public void writeStructEnd() throws TException {
    }

    @Override
    public void writeFieldBegin(TField tField) throws TException {
    }

    @Override
    public void writeFieldEnd() throws TException {
    }

    @Override
    public void writeFieldStop() throws TException {
    }

    @Override
    public void writeMapBegin(TMap tMap) throws TException {
    }

    @Override
    public void writeMapEnd() throws TException {
    }

    @Override
    public void writeListBegin(TList tList) throws TException {
    }

    @Override
    public void writeListEnd() throws TException {
    }

    @Override
    public void writeSetBegin(TSet tSet) throws TException {
    }

    @Override
    public void writeSetEnd() throws TException {
    }

    @Override
    public void writeBool(boolean b) throws TException {
    }

    @Override
    public void writeByte(byte b) throws TException {
    }

    @Override
    public void writeI16(short i) throws TException {
    }

    @Override
    public void writeI32(int i) throws TException {
    }

    @Override
    public void writeI64(long l) throws TException {
    }

    @Override
    public void writeDouble(double v) throws TException {
    }

    @Override
    public void writeString(String s) throws TException {
    }

    @Override
    public void writeBinary(ByteBuffer byteBuffer) throws TException {
    }

    @Override
    public TMessage readMessageBegin() throws TException {
      return null;
    }

    @Override
    public void readMessageEnd() throws TException {
    }

    @Override
    public TStruct readStructBegin() throws TException {
      return null;
    }

    @Override
    public void readStructEnd() throws TException {
    }

    @Override
    public TField readFieldBegin() throws TException {
      return null;
    }

    @Override
    public void readFieldEnd() throws TException {
    }

    @Override
    public TMap readMapBegin() throws TException {
      return null;
    }

    @Override
    public void readMapEnd() throws TException {
    }

    @Override
    public TList readListBegin() throws TException {
      return null;
    }

    @Override
    public void readListEnd() throws TException {
    }

    @Override
    public TSet readSetBegin() throws TException {
      return null;
    }

    @Override
    public void readSetEnd() throws TException {
    }

    @Override
    public boolean readBool() throws TException {
      return false;
    }

    @Override
    public byte readByte() throws TException {
      return 0;
    }

    @Override
    public short readI16() throws TException {
      return 0;
    }

    @Override
    public int readI32() throws TException {
      return 0;
    }

    @Override
    public long readI64() throws TException {
      return 0;
    }

    @Override
    public double readDouble() throws TException {
      return 0;
    }

    @Override
    public String readString() throws TException {
      return null;
    }

    @Override
    public ByteBuffer readBinary() throws TException {
      return null;
    }
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop.api;

import java.util.Map;

import org.apache.hadoop.conf.Configuration;

import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;

/**
 * Abstraction used by the {@link org.apache.parquet.hadoop.ParquetInputFormat} to materialize records
 *
 * @param <T> the type of the materialized record
 */
abstract public class ReadSupport<T> {

  /**
   * configuration key for a parquet read projection schema
   */
	public static final String PARQUET_READ_SCHEMA = "parquet.read.schema";

  /**
   * attempts to validate and construct a {@link MessageType} from a read projection schema
   *
   * @param fileMessageType         the typed schema of the source
   * @param partialReadSchemaString the requested projection schema
   * @return the typed schema that should be used to read
   */
  public static MessageType getSchemaForRead(MessageType fileMessageType, String partialReadSchemaString) {
    if (partialReadSchemaString == null)
      return fileMessageType;
    MessageType requestedMessageType = MessageTypeParser.parseMessageType(partialReadSchemaString);
    return getSchemaForRead(fileMessageType, requestedMessageType);
  }

  public static MessageType getSchemaForRead(MessageType fileMessageType, MessageType projectedMessageType) {
    fileMessageType.checkContains(projectedMessageType);
    return projectedMessageType;
  }

  /**
   * called in {@link org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)} in the front end
   *
   * @param configuration    the job configuration
   * @param keyValueMetaData the app specific metadata from the file
   * @param fileSchema       the schema of the file
   * @return the readContext that defines how to read the file
   *
   * @deprecated override {@link ReadSupport#init(InitContext)} instead
   */
  @Deprecated
  public ReadContext init(
          Configuration configuration,
          Map<String, String> keyValueMetaData,
          MessageType fileSchema) {
    throw new UnsupportedOperationException("Override init(InitContext)");
  }

  /**
   * called in {@link org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)} in the front end
   *
   * @param context the initialisation context
   * @return the readContext that defines how to read the file
   */
  public ReadContext init(InitContext context) {
    return init(context.getConfiguration(), context.getMergedKeyValueMetaData(), context.getFileSchema());
  }

  /**
   * called in {@link org.apache.hadoop.mapreduce.RecordReader#initialize(org.apache.hadoop.mapreduce.InputSplit, org.apache.hadoop.mapreduce.TaskAttemptContext)} in the back end
   * the returned RecordMaterializer will materialize the records and add them to the destination
   *
   * @param configuration    the job configuration
   * @param keyValueMetaData the app specific metadata from the file
   * @param fileSchema       the schema of the file
   * @param readContext      returned by the init method
   * @return the recordMaterializer that will materialize the records
   */
  abstract public RecordMaterializer<T> prepareForRead(
          Configuration configuration,
          Map<String, String> keyValueMetaData,
          MessageType fileSchema,
          ReadContext readContext);

  /**
   * information to read the file
   */
  public static final class ReadContext {
    private final MessageType requestedSchema;
    private final Map<String, String> readSupportMetadata;

    /**
     * @param requestedSchema the schema requested by the user. Can not be null.
     */
    public ReadContext(MessageType requestedSchema) {
      this(requestedSchema, null);
    }

    /**
     * @param requestedSchema the schema requested by the user. Can not be null.
     * @param readSupportMetadata metadata specific to the ReadSupport implementation. Will be available in the prepareForRead phase.
     */
    public ReadContext(MessageType requestedSchema, Map<String, String> readSupportMetadata) {
      super();
      if (requestedSchema == null) {
        throw new NullPointerException("requestedSchema");
      }
      this.requestedSchema = requestedSchema;
      this.readSupportMetadata = readSupportMetadata;
    }

    /**
     * @return the schema of the file
     */
    public MessageType getRequestedSchema() {
      return requestedSchema;
    }

    /**
     * @return metadata specific to the ReadSupport implementation
     */
    public Map<String, String> getReadSupportMetadata() {
      return readSupportMetadata;
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.statistics;

import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Types;

public class BinaryStatistics extends Statistics<Binary> {

  // A fake type object to be used to generate the proper comparator
  private static final PrimitiveType DEFAULT_FAKE_TYPE = Types.optional(PrimitiveType.PrimitiveTypeName.BINARY)
      .named("fake_binary_type");

  private Binary max;
  private Binary min;

  /**
   * @deprecated will be removed in 2.0.0. Use {@link Statistics#createStats(org.apache.parquet.schema.Type)} instead
   */
  @Deprecated
  public BinaryStatistics() {
    this(DEFAULT_FAKE_TYPE);
  }

  BinaryStatistics(PrimitiveType type) {
    super(type);
  }

  private BinaryStatistics(BinaryStatistics other) {
    super(other.type());
    if (other.hasNonNullValue()) {
      initializeStats(other.min, other.max);
    }
    setNumNulls(other.getNumNulls());
  }

  @Override
  public void updateStats(Binary value) {
    if (!this.hasNonNullValue()) {
      initializeStats(value, value);
    } else {
      updateStats(value, value);
    }
  }

  @Override
  public void mergeStatisticsMinMax(Statistics stats) {
    BinaryStatistics binaryStats = (BinaryStatistics)stats;
    if (!this.hasNonNullValue()) {
      initializeStats(binaryStats.getMin(), binaryStats.getMax());
    } else {
      updateStats(binaryStats.getMin(), binaryStats.getMax());
    }
  }

  /**
   * Sets min and max values, re-uses the byte[] passed in.
   * Any changes made to byte[] will be reflected in min and max values as well.
   * @param minBytes byte array to set the min value to
   * @param maxBytes byte array to set the max value to
   */
  @Override
  public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
    max = Binary.fromReusedByteArray(maxBytes);
    min = Binary.fromReusedByteArray(minBytes);
    this.markAsNotEmpty();
  }

  @Override
  public byte[] getMaxBytes() {
    return max == null ? null : max.getBytes();
  }

  @Override
  public byte[] getMinBytes() {
    return min == null ? null : min.getBytes();
  }

  @Override
  String stringify(Binary value) {
    return stringifier.stringify(value);
  }

  @Override
  public boolean isSmallerThan(long size) {
    return !hasNonNullValue() || ((min.length() + max.length()) < size);
  }

  /**
   * @param min_value a min binary
   * @param max_value a max binary
   * @deprecated use {@link #updateStats(Binary)}, will be removed in 2.0.0
   */
  @Deprecated
  public void updateStats(Binary min_value, Binary max_value) {
    if (comparator().compare(min, min_value) > 0) { min = min_value.copy(); }
    if (comparator().compare(max, max_value) < 0) { max = max_value.copy(); }
  }

  /**
   * @param min_value a min binary
   * @param max_value a max binary
   * @deprecated use {@link #updateStats(Binary)}, will be removed in 2.0.0
   */
  @Deprecated
  public void initializeStats(Binary min_value, Binary max_value) {
      min = min_value.copy();
      max = max_value.copy();
      this.markAsNotEmpty();
  }

  @Override
  public Binary genericGetMin() {
    return min;
  }

  @Override
  public Binary genericGetMax() {
    return max;
  }

  /**
   * @return the max binary
   * @deprecated use {@link #genericGetMax()}, will be removed in 2.0.0
   */
  @Deprecated
  public Binary getMax() {
    return max;
  }

  /**
   * @return the min binary
   * @deprecated use {@link #genericGetMin()}, will be removed in 2.0.0
   */
  @Deprecated
  public Binary getMin() {
    return min;
  }

  /**
   * @param min a min binary
   * @param max a max binary
   * @deprecated use {@link #updateStats(Binary)}, will be removed in 2.0.0
   */
  @Deprecated
  public void setMinMax(Binary min, Binary max) {
    this.max = max;
    this.min = min;
    this.markAsNotEmpty();
  }

  @Override
  public BinaryStatistics copy() {
    return new BinaryStatistics(this);
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.delta;

import java.io.IOException;

import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.values.bitpacking.BytePacker;
import org.apache.parquet.column.values.bitpacking.Packer;
import org.apache.parquet.io.ParquetEncodingException;

/**
 * Write integers (INT32) with delta encoding and binary packing.
 */
public class DeltaBinaryPackingValuesWriterForInteger extends DeltaBinaryPackingValuesWriter {
  /**
   * max bitwidth for a mini block, it is used to allocate miniBlockByteBuffer which is
   * reused between flushes.
   */
  private static final int MAX_BITWIDTH = 32;

  /**
   * stores delta values starting from the 2nd value written(1st value is stored in header).
   * It's reused between flushes
   */
  private int[] deltaBlockBuffer;

  /**
   * firstValue is written to the header of the page
   */
  private int firstValue = 0;

  /**
   * cache previous written value for calculating delta
   */
  private int previousValue = 0;

  /**
   * min delta is written to the beginning of each block.
   * it's zig-zag encoded. The deltas stored in each block is actually the difference to min delta,
   * therefore are all positive
   * it will be reset after each flush
   */
  private int minDeltaInCurrentBlock = Integer.MAX_VALUE;

  public DeltaBinaryPackingValuesWriterForInteger(
      int slabSize, int pageSize, ByteBufferAllocator allocator) {
    this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, slabSize, pageSize, allocator);
  }

  public DeltaBinaryPackingValuesWriterForInteger(int blockSizeInValues, int miniBlockNum,
      int slabSize, int pageSize, ByteBufferAllocator allocator) {
    super(blockSizeInValues, miniBlockNum, slabSize, pageSize, allocator);
    deltaBlockBuffer = new int[config.blockSizeInValues];
    miniBlockByteBuffer = new byte[config.miniBlockSizeInValues * MAX_BITWIDTH];
  }

  @Override
  public void writeInteger(int v) {
    totalValueCount++;

    if (totalValueCount == 1) {
      firstValue = v;
      previousValue = firstValue;
      return;
    }

    // Calculate delta. The possible overflow is accounted for. The algorithm is correct because
    // Java int is working as a modalar ring with base 2^32 and because of the plus and minus
    // properties of a ring. http://en.wikipedia.org/wiki/Modular_arithmetic#Integers_modulo_n
    int delta = v - previousValue;
    previousValue = v;

    deltaBlockBuffer[deltaValuesToFlush++] = delta;

    if (delta < minDeltaInCurrentBlock) {
      minDeltaInCurrentBlock = delta;
    }

    if (config.blockSizeInValues == deltaValuesToFlush) {
      flushBlockBuffer();
    }
  }

  private void flushBlockBuffer() {
    // since we store the min delta, the deltas will be converted to be the difference to min delta
    // and all positive
    for (int i = 0; i < deltaValuesToFlush; i++) {
      deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaInCurrentBlock;
    }

    writeMinDelta();
    int miniBlocksToFlush = getMiniBlockCountToFlush(deltaValuesToFlush);

    calculateBitWidthsForDeltaBlockBuffer(miniBlocksToFlush);
    for (int i = 0; i < config.miniBlockNumInABlock; i++) {
      writeBitWidthForMiniBlock(i);
    }

    for (int i = 0; i < miniBlocksToFlush; i++) {
      // writing i th miniblock
      int currentBitWidth = bitWidths[i];
      int blockOffset = 0;
      BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(currentBitWidth);
      int miniBlockStart = i * config.miniBlockSizeInValues;
      for (int j = miniBlockStart; j < (i + 1) * config.miniBlockSizeInValues; j += 8) {//8 values per pack
        // mini block is atomic in terms of flushing
        // This may write more values when reach to the end of data writing to last mini block,
        // since it may not be aligned to miniblock,
        // but doesn't matter. The reader uses total count to see if reached the end.
        packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset);
        blockOffset += currentBitWidth;
      }
      baos.write(miniBlockByteBuffer, 0, blockOffset);
    }

    minDeltaInCurrentBlock = Integer.MAX_VALUE;
    deltaValuesToFlush = 0;
  }

  private void writeMinDelta() {
    try {
      BytesUtils.writeZigZagVarInt(minDeltaInCurrentBlock, baos);
    } catch (IOException e) {
      throw new ParquetEncodingException("can not write min delta for block", e);
    }
  }

  /**
   * iterate through values in each mini block and calculate the bitWidths of max values.
   *
   * @param miniBlocksToFlush number of miniblocks
   */
  private void calculateBitWidthsForDeltaBlockBuffer(int miniBlocksToFlush) {
    for (int miniBlockIndex = 0; miniBlockIndex < miniBlocksToFlush; miniBlockIndex++) {
      int mask = 0;
      int miniStart = miniBlockIndex * config.miniBlockSizeInValues;

      //The end of current mini block could be the end of current block(deltaValuesToFlush) buffer when data is not aligned to mini block
      int miniEnd = Math.min((miniBlockIndex + 1) * config.miniBlockSizeInValues, deltaValuesToFlush);

      for (int i = miniStart; i < miniEnd; i++) {
        mask |= deltaBlockBuffer[i];
      }
      bitWidths[miniBlockIndex] = 32 - Integer.numberOfLeadingZeros(mask);
    }
  }

  /**
   * getBytes will trigger flushing block buffer, DO NOT write after getBytes() is called without calling reset()
   *
   * @return a BytesInput that contains the encoded page data
   */
  @Override
  public BytesInput getBytes() {
    // The Page Header should include: blockSizeInValues, numberOfMiniBlocks, totalValueCount
    if (deltaValuesToFlush != 0) {
      flushBlockBuffer();
    }
    return BytesInput.concat(
            config.toBytesInput(),
            BytesInput.fromUnsignedVarInt(totalValueCount),
            BytesInput.fromZigZagVarInt(firstValue),
            BytesInput.from(baos));
  }

  @Override
  public void reset() {
    super.reset();
    this.minDeltaInCurrentBlock = Integer.MAX_VALUE;
  }

  @Override
  public void close() {
    super.close();
    this.minDeltaInCurrentBlock = Integer.MAX_VALUE;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter;

import org.apache.parquet.Preconditions;
import org.apache.parquet.column.ColumnReader;

/**
 * Provides ability to chain two filters together. Bear in mind that the first one will
 * short circuit the second. Useful if getting a page of already filtered result.
 *  i.e and( column("manufacturer", equalTo("Volkswagen")), page(100,50))
 */
public final class AndRecordFilter implements RecordFilter {

  private final RecordFilter boundFilter1;
  private final RecordFilter boundFilter2;

  /**
   * Returns builder for creating an and filter.
   * @param filter1 The first filter to check.
   * @param filter2 The second filter to check.
   * @return an unbound and filter
   */
  public static final UnboundRecordFilter and( final UnboundRecordFilter filter1, final UnboundRecordFilter filter2 ) {
    Preconditions.checkNotNull( filter1, "filter1" );
    Preconditions.checkNotNull( filter2, "filter2" );
    return new UnboundRecordFilter() {
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        return new AndRecordFilter( filter1.bind(readers), filter2.bind( readers) );
      }
    };
  }

  /**
   * Private constructor, use AndRecordFilter.and() instead.
   */
  private AndRecordFilter( RecordFilter boundFilter1, RecordFilter boundFilter2 ) {
    this.boundFilter1 = boundFilter1;
    this.boundFilter2 = boundFilter2;
  }

  @Override
  public boolean isMatch() {
    return boundFilter1.isMatch() && boundFilter2.isMatch();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.schema;

import java.util.ArrayList;
import java.util.List;

import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.io.InvalidRecordException;

/**
 * The root of a schema
 */
public final class MessageType extends GroupType {

  /**
   *
   * @param name the name of the type
   * @param fields the fields contained by this message
   */
  public MessageType(String name, Type... fields) {
    super(Repetition.REPEATED, name, fields);
  }

 /**
  *
  * @param name the name of the type
  * @param fields the fields contained by this message
  */
 public MessageType(String name, List<Type> fields) {
   super(Repetition.REPEATED, name, fields);
 }

  /**
   * {@inheritDoc}
   */
  @Override
  public void accept(TypeVisitor visitor) {
    visitor.visit(this);
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public void writeToStringBuilder(StringBuilder sb, String indent) {
    sb.append("message ")
        .append(getName())
        .append(getLogicalTypeAnnotation() == null ? "" : " (" + getLogicalTypeAnnotation().toString() +")")
        .append(" {\n");
    membersDisplayString(sb, "  ");
    sb.append("}\n");
  }

  /**
   * @param path an array of strings representing the name path in this type
   * @return the max repetition level that might be needed to encode the
   * type at 'path'.
   */
  public int getMaxRepetitionLevel(String ... path) {
    return getMaxRepetitionLevel(path, 0) - 1;
  }

  /**
   * @param path an array of strings representing the name path in this type
   * @return the max repetition level that might be needed to encode the
   * type at 'path'.
   */
  public int getMaxDefinitionLevel(String ... path) {
    return getMaxDefinitionLevel(path, 0) - 1;
  }

  public Type getType(String ... path) {
    return getType(path, 0);
  }

  public ColumnDescriptor getColumnDescription(String[] path) {
    int maxRep = getMaxRepetitionLevel(path);
    int maxDef = getMaxDefinitionLevel(path);
    PrimitiveType type = getType(path).asPrimitiveType();
    return new ColumnDescriptor(path, type, maxRep, maxDef);
  }

  public List<String[]> getPaths() {
    return this.getPaths(0);
  }

  public List<ColumnDescriptor> getColumns() {
    List<String[]> paths = this.getPaths(0);
    List<ColumnDescriptor> columns = new ArrayList<ColumnDescriptor>(paths.size());
    for (String[] path : paths) {
      // TODO: optimize this
      PrimitiveType primitiveType = getType(path).asPrimitiveType();
      columns.add(new ColumnDescriptor(
                      path,
                      primitiveType,
                      getMaxRepetitionLevel(path),
                      getMaxDefinitionLevel(path)));
    }
    return columns;
  }

  @Override
  public void checkContains(Type subType) {
    if (!(subType instanceof MessageType)) {
      throw new InvalidRecordException(subType + " found: expected " + this);
    }
    checkGroupContains(subType);
  }

  public <T> T convertWith(TypeConverter<T> converter) {
    final ArrayList<GroupType> path = new ArrayList<GroupType>();
    path.add(this);
    return converter.convertMessageType(this, convertChildren(path, converter));
  }

  public boolean containsPath(String[] path) {
    return containsPath(path, 0);
  }

  public MessageType union(MessageType toMerge) {
    return union(toMerge, true);
  }

  public MessageType union(MessageType toMerge, boolean strict) {
    return new MessageType(this.getName(), mergeFields(toMerge, strict));
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hadoop;

import static java.util.Collections.emptySet;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.internal.column.columnindex.ColumnIndex;
import org.apache.parquet.internal.column.columnindex.OffsetIndex;
import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Internal implementation of {@link ColumnIndexStore}.
 */
class ColumnIndexStoreImpl implements ColumnIndexStore {

  private interface IndexStore {
    ColumnIndex getColumnIndex();

    OffsetIndex getOffsetIndex();
  }

  private class IndexStoreImpl implements IndexStore {
    private final ColumnChunkMetaData meta;
    private ColumnIndex columnIndex;
    private boolean columnIndexRead;
    private final OffsetIndex offsetIndex;

    IndexStoreImpl(ColumnChunkMetaData meta) {
      this.meta = meta;
      OffsetIndex oi;
      try {
        oi = reader.readOffsetIndex(meta);
      } catch (IOException e) {
        // If the I/O issue still stands it will fail the reading later;
        // otherwise we fail the filtering only with a missing offset index.
        LOGGER.warn("Unable to read offset index for column {}", meta.getPath(), e);
        oi = null;
      }
      if (oi == null) {
        throw new MissingOffsetIndexException(meta.getPath());
      }
      offsetIndex = oi;
    }

    @Override
    public ColumnIndex getColumnIndex() {
      if (!columnIndexRead) {
        try {
          columnIndex = reader.readColumnIndex(meta);
        } catch (IOException e) {
          // If the I/O issue still stands it will fail the reading later;
          // otherwise we fail the filtering only with a missing column index.
          LOGGER.warn("Unable to read column index for column {}", meta.getPath(), e);
        }
        columnIndexRead = true;
      }
      return columnIndex;
    }

    @Override
    public OffsetIndex getOffsetIndex() {
      return offsetIndex;
    }
  }

  private static final Logger LOGGER = LoggerFactory.getLogger(ColumnIndexStoreImpl.class);
  // Used for columns are not in this parquet file
  private static final IndexStore MISSING_INDEX_STORE = new IndexStore() {
    @Override
    public ColumnIndex getColumnIndex() {
      return null;
    }

    @Override
    public OffsetIndex getOffsetIndex() {
      return null;
    }
  };
  private static final ColumnIndexStoreImpl EMPTY = new ColumnIndexStoreImpl(null, new BlockMetaData(), emptySet()) {
    @Override
    public ColumnIndex getColumnIndex(ColumnPath column) {
      return null;
    }

    @Override
    public OffsetIndex getOffsetIndex(ColumnPath column) {
      throw new MissingOffsetIndexException(column);
    }
  };

  private final ParquetFileReader reader;
  private final Map<ColumnPath, IndexStore> store;

  /*
   * Creates a column index store which lazily reads column/offset indexes for the columns in paths. (paths are the set
   * of columns used for the projection)
   */
  static ColumnIndexStore create(ParquetFileReader reader, BlockMetaData block, Set<ColumnPath> paths) {
    try {
      return new ColumnIndexStoreImpl(reader, block, paths);
    } catch (MissingOffsetIndexException e) {
      return EMPTY;
    }
  }

  private ColumnIndexStoreImpl(ParquetFileReader reader, BlockMetaData block, Set<ColumnPath> paths) {
    // TODO[GS]: Offset index for every paths will be required; pre-read the consecutive ones at once?
    // TODO[GS]: Pre-read column index based on filter?
    this.reader = reader;
    Map<ColumnPath, IndexStore> store = new HashMap<>();
    for (ColumnChunkMetaData column : block.getColumns()) {
      ColumnPath path = column.getPath();
      if (paths.contains(path)) {
        store.put(path, new IndexStoreImpl(column));
      }
    }
    this.store = store;
  }

  @Override
  public ColumnIndex getColumnIndex(ColumnPath column) {
    return store.getOrDefault(column, MISSING_INDEX_STORE).getColumnIndex();
  }

  @Override
  public OffsetIndex getOffsetIndex(ColumnPath column) {
    return store.getOrDefault(column, MISSING_INDEX_STORE).getOffsetIndex();
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter;

import org.apache.parquet.Preconditions;
import org.apache.parquet.column.ColumnReader;

/**
 * Provides ability to negate the result of a filter.
 */
public final class NotRecordFilter implements RecordFilter {

  private final RecordFilter boundFilter;

  /**
   * Returns builder for creating an and filter.
   * @param filter The filter to invert.
   * @return a not record filter
   */
  public static final UnboundRecordFilter not( final UnboundRecordFilter filter) {
    Preconditions.checkNotNull( filter, "filter" );
    return new UnboundRecordFilter() {
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        return new NotRecordFilter( filter.bind(readers) );
      }
    };
  }

  /**
   * Private constructor, use NotRecordFilter.not() instead.
   */
  private NotRecordFilter( RecordFilter boundFilter) {
    this.boundFilter = boundFilter;
  }

  @Override
  public boolean isMatch() {
    return !(boundFilter.isMatch());
  }
}


/*
 *  Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 */

package org.apache.parquet;

public class Exceptions {
  /**
   * If the given throwable is an instance of E, throw it as an E.
   *
   * @param t an exception instance
   * @param excClass an exception class t may be an instance of
   * @param <E> the type of exception that will be thrown if throwable is an instance
   * @throws E if t is an instance of E
   */
  public static <E extends Exception> void throwIfInstance(Throwable t,
                                                           Class<E> excClass)
      throws E {
    if (excClass.isAssignableFrom(t.getClass())) {
      // the throwable is already an exception, so return it
      throw excClass.cast(t);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.statistics;

import java.util.Arrays;
import org.apache.parquet.column.UnknownColumnTypeException;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.PrimitiveComparator;
import org.apache.parquet.schema.PrimitiveStringifier;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;

/**
 * Statistics class to keep track of statistics in parquet pages and column chunks
 *
 * @param <T> the Java type described by this Statistics instance
 */
public abstract class Statistics<T extends Comparable<T>> {

  /**
   * Builder class to build Statistics objects. Used to read the statistics from the Parquet file.
   */
  public static class Builder {
    private final PrimitiveType type;
    private byte[] min;
    private byte[] max;
    private long numNulls = -1;

    private Builder(PrimitiveType type) {
      this.type = type;
    }

    public Builder withMin(byte[] min) {
      this.min = min;
      return this;
    }

    public Builder withMax(byte[] max) {
      this.max = max;
      return this;
    }

    public Builder withNumNulls(long numNulls) {
      this.numNulls = numNulls;
      return this;
    }

    public Statistics<?> build() {
      Statistics<?> stats = createStats(type);
      if (min != null && max != null) {
        stats.setMinMaxFromBytes(min, max);
      }
      stats.num_nulls = this.numNulls;
      return stats;
    }
  }

  // Builder for FLOAT type to handle special cases of min/max values like NaN, -0.0, and 0.0
  private static class FloatBuilder extends Builder {
    public FloatBuilder(PrimitiveType type) {
      super(type);
      assert type.getPrimitiveTypeName() == PrimitiveTypeName.FLOAT;
    }

    @Override
    public Statistics<?> build() {
      FloatStatistics stats = (FloatStatistics) super.build();
      if (stats.hasNonNullValue()) {
        Float min = stats.genericGetMin();
        Float max = stats.genericGetMax();
        // Drop min/max values in case of NaN as the sorting order of values is undefined for this case
        if (min.isNaN() || max.isNaN()) {
          stats.setMinMax(0.0f, 0.0f);
          ((Statistics<?>) stats).hasNonNullValue = false;
        } else {
          // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped
          if (Float.compare(min, 0.0f) == 0) {
            min = -0.0f;
            stats.setMinMax(min, max);
          }
          if (Float.compare(max, -0.0f) == 0) {
            max = 0.0f;
            stats.setMinMax(min, max);
          }
        }
      }
      return stats;
    }
  }

  // Builder for DOUBLE type to handle special cases of min/max values like NaN, -0.0, and 0.0
  private static class DoubleBuilder extends Builder {
    public DoubleBuilder(PrimitiveType type) {
      super(type);
      assert type.getPrimitiveTypeName() == PrimitiveTypeName.DOUBLE;
    }

    @Override
    public Statistics<?> build() {
      DoubleStatistics stats = (DoubleStatistics) super.build();
      if (stats.hasNonNullValue()) {
        Double min = stats.genericGetMin();
        Double max = stats.genericGetMax();
        // Drop min/max values in case of NaN as the sorting order of values is undefined for this case
        if (min.isNaN() || max.isNaN()) {
          stats.setMinMax(0.0, 0.0);
          ((Statistics<?>) stats).hasNonNullValue = false;
        } else {
          // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped
          if (Double.compare(min, 0.0) == 0) {
            min = -0.0;
            stats.setMinMax(min, max);
          }
          if (Double.compare(max, -0.0) == 0) {
            max = 0.0;
            stats.setMinMax(min, max);
          }
        }
      }
      return stats;
    }
  }

  private final PrimitiveType type;
  private final PrimitiveComparator<T> comparator;
  private boolean hasNonNullValue;
  private long num_nulls;
  final PrimitiveStringifier stringifier;

  Statistics(PrimitiveType type) {
    this.type = type;
    this.comparator = type.comparator();
    this.stringifier = type.stringifier();
    hasNonNullValue = false;
    num_nulls = 0;
  }

  /**
   * Returns the typed statistics object based on the passed type parameter
   * @param type PrimitiveTypeName type of the column
   * @return instance of a typed statistics class
   * @deprecated Use {@link #createStats(Type)} instead
   */
  @Deprecated
  public static Statistics getStatsBasedOnType(PrimitiveTypeName type) {
    switch (type) {
      case INT32:
        return new IntStatistics();
      case INT64:
        return new LongStatistics();
      case FLOAT:
        return new FloatStatistics();
      case DOUBLE:
        return new DoubleStatistics();
      case BOOLEAN:
        return new BooleanStatistics();
      case BINARY:
        return new BinaryStatistics();
      case INT96:
        return new BinaryStatistics();
      case FIXED_LEN_BYTE_ARRAY:
        return new BinaryStatistics();
      default:
        throw new UnknownColumnTypeException(type);
    }
  }

  /**
   * Creates an empty {@code Statistics} instance for the specified type to be
   * used for reading/writing the new min/max statistics used in the V2 format.
   *
   * @param type
   *          type of the column
   * @return instance of a typed statistics class
   */
  public static Statistics<?> createStats(Type type) {
    PrimitiveType primitive = type.asPrimitiveType();
    switch (primitive.getPrimitiveTypeName()) {
      case INT32:
        return new IntStatistics(primitive);
      case INT64:
        return new LongStatistics(primitive);
      case FLOAT:
        return new FloatStatistics(primitive);
      case DOUBLE:
        return new DoubleStatistics(primitive);
      case BOOLEAN:
        return new BooleanStatistics(primitive);
      case BINARY:
      case INT96:
      case FIXED_LEN_BYTE_ARRAY:
        return new BinaryStatistics(primitive);
      default:
        throw new UnknownColumnTypeException(primitive.getPrimitiveTypeName());
    }
  }

  /**
   * Returns a builder to create new statistics object. Used to read the statistics from the parquet file.
   *
   * @param type
   *          type of the column
   * @return builder to create new statistics object
   */
  public static Builder getBuilderForReading(PrimitiveType type) {
    switch (type.getPrimitiveTypeName()) {
      case FLOAT:
        return new FloatBuilder(type);
      case DOUBLE:
        return new DoubleBuilder(type);
      default:
        return new Builder(type);
    }
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(int value) {
    throw new UnsupportedOperationException();
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(long value) {
    throw new UnsupportedOperationException();
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(float value) {
    throw new UnsupportedOperationException();
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(double value) {
    throw new UnsupportedOperationException();
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(boolean value) {
    throw new UnsupportedOperationException();
  }

  /**
   * updates statistics min and max using the passed value
   * @param value value to use to update min and max
   */
  public void updateStats(Binary value) {
    throw new UnsupportedOperationException();
  }

  /**
   * Equality comparison method to compare two statistics objects.
   * @param other Object to compare against
   * @return true if objects are equal, false otherwise
   */
  @Override
  public boolean equals(Object other) {
    if (other == this)
      return true;
    if (!(other instanceof Statistics))
      return false;
    Statistics stats = (Statistics) other;
    return type.equals(stats.type) &&
        Arrays.equals(stats.getMaxBytes(), this.getMaxBytes()) &&
        Arrays.equals(stats.getMinBytes(), this.getMinBytes()) &&
        stats.getNumNulls() == this.getNumNulls();
  }

  /**
   * Hash code for the statistics object
   * @return hash code int
   */
  @Override
  public int hashCode() {
    return 31 * type.hashCode() + 31 * Arrays.hashCode(getMaxBytes()) + 17 * Arrays.hashCode(getMinBytes())
        + Long.valueOf(this.getNumNulls()).hashCode();
  }

  /**
   * Method to merge this statistics object with the object passed
   * as parameter. Merging keeps the smallest of min values, largest of max
   * values and combines the number of null counts.
   * @param stats Statistics object to merge with
   */
  public void mergeStatistics(Statistics stats) {
    if (stats.isEmpty()) return;

    // Merge stats only if they have the same type
    if (type.equals(stats.type)) {
      incrementNumNulls(stats.getNumNulls());
      if (stats.hasNonNullValue()) {
        mergeStatisticsMinMax(stats);
        markAsNotEmpty();
      }
    } else {
      throw StatisticsClassException.create(this, stats);
    }
  }

  /**
   * Abstract method to merge this statistics min and max with the values
   * of the parameter object. Does not do any checks, only called internally.
   * @param stats Statistics object to merge with
   */
  abstract protected void mergeStatisticsMinMax(Statistics stats);

  /**
   * Abstract method to set min and max values from byte arrays.
   * @param minBytes byte array to set the min value to
   * @param maxBytes byte array to set the max value to
   * @deprecated will be removed in 2.0.0. Use {@link #getBuilderForReading(PrimitiveType)} instead.
   */
  @Deprecated
  abstract public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes);

  /**
   * Returns the min value in the statistics. The java natural order of the returned type defined by {@link
   * Comparable#compareTo(Object)} might not be the proper one. For example, UINT_32 requires unsigned comparison instead of the
   * natural signed one. Use {@link #compareMinToValue(Comparable)} or the comparator returned by {@link #comparator()} to
   * always get the proper ordering.
   *
   * @return the min value
   */
  abstract public T genericGetMin();

  /**
   * Returns the max value in the statistics. The java natural order of the returned type defined by {@link
   * Comparable#compareTo(Object)} might not be the proper one. For example, UINT_32 requires unsigned comparison instead of the
   * natural signed one. Use {@link #compareMaxToValue(Comparable)} or the comparator returned by {@link #comparator()} to
   * always get the proper ordering.
   *
   * @return the max value
   */
  abstract public T genericGetMax();

  /**
   * Returns the {@link PrimitiveComparator} implementation to be used to compare two generic values in the proper way
   * (for example, unsigned comparison for UINT_32).
   *
   * @return the comparator for data described by this Statistics instance
   */
  public final PrimitiveComparator<T> comparator() {
    return comparator;
  }

  /**
   * Compares min to the specified value in the proper way. It does the same as invoking
   * {@code comparator().compare(genericGetMin(), value)}. The corresponding statistics implementations overload this
   * method so the one with the primitive argument shall be used to avoid boxing/unboxing.
   *
   * @param value
   *          the value which {@code min} is to be compared to
   * @return a negative integer, zero, or a positive integer as {@code min} is less than, equal to, or greater than
   *         {@code value}.
   */
  public final int compareMinToValue(T value) {
    return comparator.compare(genericGetMin(), value);
  }

  /**
   * Compares max to the specified value in the proper way. It does the same as invoking
   * {@code comparator().compare(genericGetMax(), value)}. The corresponding statistics implementations overload this
   * method so the one with the primitive argument shall be used to avoid boxing/unboxing.
   *
   * @param value
   *          the value which {@code max} is to be compared to
   * @return a negative integer, zero, or a positive integer as {@code max} is less than, equal to, or greater than
   *         {@code value}.
   */
  public final int compareMaxToValue(T value) {
    return comparator.compare(genericGetMax(), value);
  }

  /**
   * Abstract method to return the max value as a byte array
   * @return byte array corresponding to the max value
   */
  abstract public byte[] getMaxBytes();

  /**
   * Abstract method to return the min value as a byte array
   * @return byte array corresponding to the min value
   */
  abstract public byte[] getMinBytes();

  /**
   * Returns the string representation of min for debugging/logging purposes.
   *
   * @return the min value as a string
   */
  public String minAsString() {
    return stringify(genericGetMin());
  }

  /**
   * Returns the string representation of max for debugging/logging purposes.
   *
   * @return the max value as a string
   */
  public String maxAsString() {
    return stringify(genericGetMax());
  }

  abstract String stringify(T value);

  /**
   * Abstract method to return whether the min and max values fit in the given
   * size.
   * @param size a size in bytes
   * @return true iff the min and max values are less than size bytes
   */
  abstract public boolean isSmallerThan(long size);

  @Override
  public String toString() {
    if (this.hasNonNullValue()) {
      if (isNumNullsSet()) {
        return String.format("min: %s, max: %s, num_nulls: %d", minAsString(), maxAsString(), this.getNumNulls());
      } else {
        return String.format("min: %s, max: %s, num_nulls not defined", minAsString(), maxAsString());
      }
    } else if (!this.isEmpty())
      return String.format("num_nulls: %d, min/max not defined", this.getNumNulls());
    else
      return "no stats for this column";
  }

  /**
   * Increments the null count by one
   */
  public void incrementNumNulls() {
    num_nulls++ ;
  }

  /**
   * Increments the null count by the parameter value
   * @param increment value to increment the null count by
   */
  public void incrementNumNulls(long increment) {
    num_nulls += increment ;
  }

  /**
   * Returns the null count
   * @return null count or {@code -1} if the null count is not set
   */
  public long getNumNulls() {
    return num_nulls;
  }

  /**
   * Sets the number of nulls to the parameter value
   *
   * @param nulls
   *          null count to set the count to
   * @deprecated will be removed in 2.0.0. Use {@link #getBuilderForReading(PrimitiveType)} instead.
   */
  @Deprecated
  public void setNumNulls(long nulls) {
    num_nulls = nulls;
  }

  /**
   * Returns a boolean specifying if the Statistics object is empty,
   * i.e does not contain valid statistics for the page/column yet
   * @return true if object is empty, false otherwise
   */
  public boolean isEmpty() {
    return !hasNonNullValue && !isNumNullsSet();
  }

  /**
   * Returns whether there have been non-null values added to this statistics
   *
   * @return true if the values contained at least one non-null value
   */
  public boolean hasNonNullValue() {
    return hasNonNullValue;
  }

  /**
   * @return whether numNulls is set and can be used
   */
  public boolean isNumNullsSet() {
    return num_nulls >= 0;
  }

  /**
   * Sets the page/column as having a valid non-null value
   * kind of misnomer here
   */
  protected void markAsNotEmpty() {
    hasNonNullValue = true;
  }

  /**
   * @return a new independent statistics instance of this class.
   */
  public abstract Statistics<T> copy();

  /**
   * @return the primitive type object which this statistics is created for
   */
  public PrimitiveType type() {
    return type;
  }
}