/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.schema;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;

import org.apache.parquet.Preconditions;
import org.apache.parquet.schema.ColumnOrder.ColumnOrderName;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type.ID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class provides fluent builders that produce Parquet schema Types.
 *

 * The most basic use is to build primitive types:
 *

 *   Types.required(INT64).named("id");
 *   Types.optional(INT32).named("number");
 *

 *

 * The required(PrimitiveTypeName) factory method produces a primitive
 * type builder, and the {@link PrimitiveBuilder#named(String)} builds the
 * {@link PrimitiveType}. Between {@code required} and {@code named}, other
 * builder methods can be used to add type annotations or other type metadata:
 *

 *   Types.required(BINARY).as(UTF8).named("username");
 *   Types.optional(FIXED_LEN_BYTE_ARRAY).length(20).named("sha1");
 *

 *

 * Optional types are built using optional(PrimitiveTypeName) to get
 * the builder.
 *

 * Groups are built similarly, using {@code requiredGroup()} (or the optional
 * version) to return a group builder. Group builders provide {@code required}
 * and {@code optional} to add primitive types, which return primitive builders
 * like the versions above.
 *

 *   // This produces:
 *   // required group User {
 *   //   required int64 id;
 *   //   optional binary email (UTF8);
 *   // }
 *   Types.requiredGroup()
 *            .required(INT64).named("id")
 *            .optional(BINARY).as(UTF8).named("email")
 *        .named("User")
 *

 *

 * When {@code required} is called on a group builder, the builder it returns
 * will add the type to the parent group when it is built and {@code named} will
 * return its parent group builder (instead of the type) so more fields can be
 * added.
 *

 * Sub-groups can be created using {@code requiredGroup()} to get a group
 * builder that will create the group type, add it to the parent builder, and
 * return the parent builder for more fields.
 *

 *   // required group User {
 *   //   required int64 id;
 *   //   optional binary email (UTF8);
 *   //   optional group address {
 *   //     required binary street (UTF8);
 *   //     required int32 zipcode;
 *   //   }
 *   // }
 *   Types.requiredGroup()
 *            .required(INT64).named("id")
 *            .optional(BINARY).as(UTF8).named("email")
 *            .optionalGroup()
 *                .required(BINARY).as(UTF8).named("street")
 *                .required(INT32).named("zipcode")
 *            .named("address")
 *        .named("User")
 *

 *

 * Maps are built similarly, using {@code requiredMap()} (or the optionalMap()
 * version) to return a map builder. Map builders provide {@code key} to add
 * a primitive as key or a {@code groupKey} to add a group as key. {@code key()}
 * returns a MapKey builder, which extends a primitive builder. On the other hand,
 * {@code groupKey()} returns a MapGroupKey builder, which extends a group builder.
 * A key in a map is always required.
 *

 * Once a key is built, a primitive map value can be built using {@code requiredValue()}
 * (or the optionalValue() version) that returns MapValue builder. A group map value
 * can be built using {@code requiredGroupValue()} (or the optionalGroupValue()
 * version) that returns MapGroupValue builder.
 *
 *

 *   // required group zipMap (MAP) {
 *   //   repeated group map (MAP_KEY_VALUE) {
 *   //     required float key
 *   //     optional int32 value
 *   //   }
 *   // }
 *   Types.requiredMap()
 *            .key(FLOAT)
 *            .optionalValue(INT32)
 *        .named("zipMap")
 *
 *
 *   // required group zipMap (MAP) {
 *   //   repeated group map (MAP_KEY_VALUE) {
 *   //     required group key {
 *   //       optional int64 first;
 *   //       required group second {
 *   //         required float inner_id_1;
 *   //         optional int32 inner_id_2;
 *   //       }
 *   //     }
 *   //     optional group value {
 *   //       optional group localGeoInfo {
 *   //         required float inner_value_1;
 *   //         optional int32 inner_value_2;
 *   //       }
 *   //       optional int32 zipcode;
 *   //     }
 *   //   }
 *   // }
 *   Types.requiredMap()
 *            .groupKey()
 *              .optional(INT64).named("id")
 *              .requiredGroup()
 *                .required(FLOAT).named("inner_id_1")
 *                .required(FLOAT).named("inner_id_2")
 *              .named("second")
 *            .optionalGroup()
 *              .optionalGroup()
 *                .required(FLOAT).named("inner_value_1")
 *                .optional(INT32).named("inner_value_2")
 *              .named("localGeoInfo")
 *              .optional(INT32).named("zipcode")
 *        .named("zipMap")
 *

 *

 * Message types are built using {@link #buildMessage()} and function just like
 * group builders.
 *

 *   // message User {
 *   //   required int64 id;
 *   //   optional binary email (UTF8);
 *   //   optional group address {
 *   //     required binary street (UTF8);
 *   //     required int32 zipcode;
 *   //   }
 *   // }
 *   Types.buildMessage()
 *            .required(INT64).named("id")
 *            .optional(BINARY).as(UTF8).named("email")
 *            .optionalGroup()
 *                .required(BINARY).as(UTF8).named("street")
 *                .required(INT32).named("zipcode")
 *            .named("address")
 *        .named("User")
 *

 *

 * These builders enforce consistency checks based on the specifications in
 * the parquet-format documentation. For example, if DECIMAL is used to annotate
 * a FIXED_LEN_BYTE_ARRAY that is not long enough for its maximum precision,
 * these builders will throw an IllegalArgumentException:
 *

 *   // throws IllegalArgumentException with message:
 *   // "FIXED(4) is not long enough to store 10 digits"
 *   Types.required(FIXED_LEN_BYTE_ARRAY).length(4)
 *        .as(DECIMAL).precision(10)
 *        .named("badDecimal");
 *

 */
public class Types {
  private static final int NOT_SET = 0;

  /**
   * A base builder for {@link Type} objects.
   *
   * @param
 The type that this builder will return from
   *          {@link #named(String)} when the type is built.
   */
  public abstract static class Builder<THIS extends Builder, P> {
    protected final P parent;
    protected final Class<? extends P> returnClass;

    protected Type.Repetition repetition = null;
    protected LogicalTypeAnnotation logicalTypeAnnotation = null;
    protected Type.ID id = null;
    private boolean repetitionAlreadySet = false;

    /**
     * Construct a type builder that returns a "parent" object when the builder
     * is finished. The {@code parent} will be returned by
     * {@link #named(String)} so that builders can be chained.
     *
     * @param parent a non-null object to return from {@link #named(String)}
     */
    protected Builder(P parent) {
      this.parent = parent;
      this.returnClass = null;
    }

    /**
     * Construct a type builder that returns the {@link Type} that was built
     * when the builder is finished. The {@code returnClass} must be the
     * expected {@code Type} class.
     *
     * @param returnClass a {@code Type} to return from {@link #named(String)}
     */
    protected Builder(Class
 returnClass) {
      Preconditions.checkArgument(Type.class.isAssignableFrom(returnClass),
          "The requested return class must extend Type");
      this.returnClass = returnClass;
      this.parent = null;
    }

    protected abstract THIS self();

    protected final THIS repetition(Type.Repetition repetition) {
      Preconditions.checkArgument(!repetitionAlreadySet,
          "Repetition has already been set");
      Preconditions.checkNotNull(repetition, "Repetition cannot be null");
      this.repetition = repetition;
      this.repetitionAlreadySet = true;
      return self();
    }

    /**
     * Adds a type annotation ({@link OriginalType}) to the type being built.
     *

     * Type annotations are used to extend the types that parquet can store, by
     * specifying how the primitive types should be interpreted. This keeps the
     * set of primitive types to a minimum and reuses parquet's efficient
     * encodings. For example, strings are stored as byte arrays (binary) with
     * a UTF8 annotation.
     *
     * @param type an {@code OriginalType}
     * @return this builder for method chaining
     *
     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding logical type instead
     */
    @Deprecated
    public THIS as(OriginalType type) {
      this.logicalTypeAnnotation = LogicalTypeAnnotation.fromOriginalType(type, null);
      return self();
    }

    protected boolean newLogicalTypeSet;

    /**
     * Adds a type annotation ({@link LogicalTypeAnnotation}) to the type being built.
     *

     * Type annotations are used to extend the types that parquet can store, by
     * specifying how the primitive types should be interpreted. This keeps the
     * set of primitive types to a minimum and reuses parquet's efficient
     * encodings. For example, strings are stored as byte arrays (binary) with
     * a UTF8 annotation.
     *
     * @param type an {@code {@link LogicalTypeAnnotation}}
     * @return this builder for method chaining
     */
    public THIS as(LogicalTypeAnnotation type) {
      this.logicalTypeAnnotation = type;
      this.newLogicalTypeSet = true;
      return self();
    }

    /**
     * adds an id annotation to the type being built.
     *

     * ids are used to capture the original id when converting from models using ids (thrift, protobufs)
     *
     * @param id the id of the field
     * @return this builder for method chaining
     */
    public THIS id(int id) {
      this.id = new ID(id);
      return self();
    }

    abstract protected Type build(String name);

    /**
     * Builds a {@link Type} and returns the parent builder, if given, or the
     * {@code Type} that was built. If returning a parent object that is a
     * GroupBuilder, the constructed type will be added to it as a field.
     *

     * Note: Any configuration for this type builder should be done
     * before calling this method.
     *
     * @param name a name for the constructed type
     * @return the parent {@code GroupBuilder} or the constructed {@code Type}
     */
    public P named(String name) {
      Preconditions.checkNotNull(name, "Name is required");
      Preconditions.checkNotNull(repetition, "Repetition is required");

      Type type = build(name);
      if (parent != null) {
        // if the parent is a BaseGroupBuilder, add type to it
        if (BaseGroupBuilder.class.isAssignableFrom(parent.getClass())) {
          BaseGroupBuilder.class.cast(parent).addField(type);
        }
        return parent;
      } else if (returnClass != null) {
        // no parent indicates that the Type object should be returned
        // the constructor check guarantees that returnClass is a Type
        return returnClass.cast(type);
      } else {
        throw new IllegalStateException(
            "[BUG] Parent and return type are null: must override named");
      }
    }

    protected OriginalType getOriginalType () {
      return logicalTypeAnnotation == null ? null : logicalTypeAnnotation.toOriginalType();
    }
  }

  public abstract static class
      BasePrimitiveBuilder
      extends Builder<THIS, P> {
    private static final Logger LOGGER = LoggerFactory.getLogger(BasePrimitiveBuilder.class);
    private static final long MAX_PRECISION_INT32 = maxPrecision(4);
    private static final long MAX_PRECISION_INT64 = maxPrecision(8);
    private static final String LOGICAL_TYPES_DOC_URL =
        "https://github.com/apache/parquet-format/blob/master/LogicalTypes.md";
    private final PrimitiveTypeName primitiveType;
    private int length = NOT_SET;
    private int precision = NOT_SET;
    private int scale = NOT_SET;
    private ColumnOrder columnOrder;

    private BasePrimitiveBuilder(P parent, PrimitiveTypeName type) {
      super(parent);
      this.primitiveType = type;
    }

    private BasePrimitiveBuilder(Class
 returnType, PrimitiveTypeName type) {
      super(returnType);
      this.primitiveType = type;
    }

    @Override
    protected abstract THIS self();

    /**
     * Adds the length for a FIXED_LEN_BYTE_ARRAY.
     *
     * @param length an int length
     * @return this builder for method chaining
     */
    public THIS length(int length) {
      this.length = length;
      return self();
    }

    private boolean precisionAlreadySet;
    private boolean scaleAlreadySet;

    /**
     * Adds the precision for a DECIMAL.
     *

     * This value is required for decimals and must be less than or equal to
     * the maximum number of base-10 digits in the underlying type. A 4-byte
     * fixed, for example, can store up to 9 base-10 digits.
     *
     * @param precision an int precision value for the DECIMAL
     * @return this builder for method chaining
     *
     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding decimal type instead
     */
    @Deprecated
    public THIS precision(int precision) {
      this.precision = precision;
      precisionAlreadySet = true;
      return self();
    }

    /**
     * Adds the scale for a DECIMAL.
     *

     * This value must be less than the maximum precision of the type and must
     * be a positive number. If not set, the default scale is 0.
     *

     * The scale specifies the number of digits of the underlying unscaled
     * that are to the right of the decimal point. The decimal interpretation
     * of values in this column is: {@code value*10^(-scale)}.
     *
     * @param scale an int scale value for the DECIMAL
     * @return this builder for method chaining
     *
     * @deprecated use {@link #as(LogicalTypeAnnotation)} with the corresponding decimal type instead
     */
    @Deprecated
    public THIS scale(int scale) {
      this.scale = scale;
      scaleAlreadySet = true;
      return self();
    }

    /**
     * Adds the column order for the primitive type.
     *

     * In case of not set the default column order is {@link ColumnOrderName#TYPE_DEFINED_ORDER} except the type
     * {@link PrimitiveTypeName#INT96} and the types annotated by {@link OriginalType#INTERVAL} where the default column
     * order is {@link ColumnOrderName#UNDEFINED}.
     *
     * @param columnOrder
     *          the column order for the primitive type
     * @return this builder for method chaining
     */
    public THIS columnOrder(ColumnOrder columnOrder) {
      this.columnOrder = columnOrder;
      return self();
    }

    @Override
    protected PrimitiveType build(String name) {
      if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitiveType) {
        Preconditions.checkArgument(length > 0,
            "Invalid FIXED_LEN_BYTE_ARRAY length: " + length);
      }

      DecimalMetadata meta = decimalMetadata();

      // validate type annotations and required metadata
      if (logicalTypeAnnotation != null) {
        logicalTypeAnnotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<Boolean>() {
          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType) {
            checkBinaryPrimitiveType(stringLogicalType);
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.JsonLogicalTypeAnnotation jsonLogicalType) {
            checkBinaryPrimitiveType(jsonLogicalType);
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonLogicalType) {
            checkBinaryPrimitiveType(bsonLogicalType);
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) {
            Preconditions.checkState(
                (primitiveType == PrimitiveTypeName.INT32) ||
                (primitiveType == PrimitiveTypeName.INT64) ||
                (primitiveType == PrimitiveTypeName.BINARY) ||
                (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY),
                "DECIMAL can only annotate INT32, INT64, BINARY, and FIXED"
            );
            if (primitiveType == PrimitiveTypeName.INT32) {
              Preconditions.checkState(
                  meta.getPrecision() <= MAX_PRECISION_INT32,
                  "INT32 cannot store " + meta.getPrecision() + " digits " +
                      "(max " + MAX_PRECISION_INT32 + ")");
            } else if (primitiveType == PrimitiveTypeName.INT64) {
              Preconditions.checkState(
                  meta.getPrecision() <= MAX_PRECISION_INT64,
                  "INT64 cannot store " + meta.getPrecision() + " digits " +
                  "(max " + MAX_PRECISION_INT64 + ")");
              if (meta.getPrecision() <= MAX_PRECISION_INT32) {
                LOGGER.warn("Decimal with {} digits is stored in an INT64, but fits in an INT32. See {}.",
                            precision, LOGICAL_TYPES_DOC_URL);
              }
            } else if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
              Preconditions.checkState(
                  meta.getPrecision() <= maxPrecision(length),
                  "FIXED(" + length + ") cannot store " + meta.getPrecision() +
                  " digits (max " + maxPrecision(length) + ")");
            }
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
            checkInt32PrimitiveType(dateLogicalType);
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
            LogicalTypeAnnotation.TimeUnit unit = timeLogicalType.getUnit();
            switch (unit) {
              case MILLIS:
                checkInt32PrimitiveType(timeLogicalType);
                break;
              case MICROS:
              case NANOS:
                checkInt64PrimitiveType(timeLogicalType);
                break;
              default:
                throw new RuntimeException("Invalid time unit: " + unit);
            }
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.IntLogicalTypeAnnotation intLogicalType) {
            int bitWidth = intLogicalType.getBitWidth();
            switch (bitWidth) {
              case 8:
              case 16:
              case 32:
                checkInt32PrimitiveType(intLogicalType);
                break;
              case 64:
                checkInt64PrimitiveType(intLogicalType);
                break;
              default:
                throw new RuntimeException("Invalid bit width: " + bitWidth);
            }
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
            checkInt64PrimitiveType(timestampLogicalType);
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation intervalLogicalType) {
            Preconditions.checkState(
                (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) &&
                (length == 12),
                "INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)");
            return Optional.of(true);
          }

          @Override
          public Optional<Boolean> visit(LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
            Preconditions.checkState(
                primitiveType == PrimitiveTypeName.BINARY,
                "ENUM can only annotate binary fields");
            return Optional.of(true);
          }

          private void checkBinaryPrimitiveType(LogicalTypeAnnotation logicalTypeAnnotation) {
            Preconditions.checkState(
                primitiveType == PrimitiveTypeName.BINARY,
              logicalTypeAnnotation.toString() + " can only annotate binary fields");
          }

          private void checkInt32PrimitiveType(LogicalTypeAnnotation logicalTypeAnnotation) {
            Preconditions.checkState(primitiveType == PrimitiveTypeName.INT32,
              logicalTypeAnnotation.toString() + " can only annotate INT32");
          }

          private void checkInt64PrimitiveType(LogicalTypeAnnotation logicalTypeAnnotation) {
            Preconditions.checkState(primitiveType == PrimitiveTypeName.INT64,
              logicalTypeAnnotation.toString() + " can only annotate INT64");
          }
        }).orElseThrow(() -> new IllegalStateException(logicalTypeAnnotation + " can not be applied to a primitive type"));
      }

      if (newLogicalTypeSet) {
        return new PrimitiveType(repetition, primitiveType, length, name, logicalTypeAnnotation, id, columnOrder);
      } else {
        return new PrimitiveType(repetition, primitiveType, length, name, getOriginalType(), meta, id, columnOrder);
      }
    }

    private static long maxPrecision(int numBytes) {
      return Math.round(                      // convert double to long
          Math.floor(Math.log10(              // number of base-10 digits
              Math.pow(2, 8 * numBytes - 1) - 1)  // max value stored in numBytes
          )
      );
    }

    protected DecimalMetadata decimalMetadata() {
      DecimalMetadata meta = null;
      if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
        LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalTypeAnnotation;
        if (newLogicalTypeSet) {
          if (scaleAlreadySet) {
            Preconditions.checkArgument(this.scale == decimalType.getScale(),
              "Decimal scale should match with the scale of the logical type");
          }
          if (precisionAlreadySet) {
            Preconditions.checkArgument(this.precision == decimalType.getPrecision(),
              "Decimal precision should match with the precision of the logical type");
          }
          scale = decimalType.getScale();
          precision = decimalType.getPrecision();
        }
        Preconditions.checkArgument(precision > 0,
            "Invalid DECIMAL precision: " + precision);
        Preconditions.checkArgument(this.scale >= 0,
            "Invalid DECIMAL scale: " + this.scale);
        Preconditions.checkArgument(this.scale <= precision,
            "Invalid DECIMAL scale: cannot be greater than precision");
        meta = new DecimalMetadata(precision, scale);
      }
      return meta;
    }
  }

  /**
   * A builder for {@link PrimitiveType} objects.
   *
   * @param
 The type that this builder will return from
   *          {@link #named(String)} when the type is built.
   */
  public static class PrimitiveBuilder
 extends BasePrimitiveBuilder {

    private PrimitiveBuilder(P parent, PrimitiveTypeName type) {
      super(parent, type);
    }

    private PrimitiveBuilder(Class
 returnType, PrimitiveTypeName type) {
      super(returnType, type);
    }

    @Override
    protected PrimitiveBuilder
 self() {
      return this;
    }
  }

  public abstract static class BaseGroupBuilder
      extends Builder<THIS, P> {
    protected final List<Type> fields;

    private BaseGroupBuilder(P parent) {
      super(parent);
      this.fields = new ArrayList<Type>();
    }

    private BaseGroupBuilder(Class
 returnType) {
      super(returnType);
      this.fields = new ArrayList<Type>();
    }

    @Override
    protected abstract THIS self();

    public PrimitiveBuilder<THIS> primitive(
        PrimitiveTypeName type, Type.Repetition repetition) {
      return new PrimitiveBuilder<THIS> (self(), type)
          .repetition(repetition);
    }

    /**
     * Returns a {@link PrimitiveBuilder} for the required primitive type
     * {@code type}.
     *
     * @param type a {@link PrimitiveTypeName}
     * @return a primitive builder for {@code type} that will return this
     *          builder for additional fields.
     */
    public PrimitiveBuilder<THIS> required(
        PrimitiveTypeName type) {
      return new PrimitiveBuilder<THIS>(self(), type)
          .repetition(Type.Repetition.REQUIRED);
    }

    /**
     * Returns a {@link PrimitiveBuilder} for the optional primitive type
     * {@code type}.
     *
     * @param type a {@link PrimitiveTypeName}
     * @return a primitive builder for {@code type} that will return this
     *          builder for additional fields.
     */
    public PrimitiveBuilder<THIS> optional(
        PrimitiveTypeName type) {
      return new PrimitiveBuilder<THIS>(self(), type)
          .repetition(Type.Repetition.OPTIONAL);
    }

    /**
     * Returns a {@link PrimitiveBuilder} for the repeated primitive type
     * {@code type}.
     *
     * @param type a {@link PrimitiveTypeName}
     * @return a primitive builder for {@code type} that will return this
     *          builder for additional fields.
     */
    public PrimitiveBuilder<THIS> repeated(
        PrimitiveTypeName type) {
      return new PrimitiveBuilder<THIS>(self(), type)
          .repetition(Type.Repetition.REPEATED);
    }

    public GroupBuilder<THIS> group(Type.Repetition repetition) {
      return new GroupBuilder<THIS>(self())
          .repetition(repetition);
    }

    /**
     * Returns a {@link GroupBuilder} to build a required sub-group.
     *
     * @return a group builder that will return this builder for additional
     *          fields.
     */
    public GroupBuilder<THIS> requiredGroup() {
      return new GroupBuilder<THIS>(self())
          .repetition(Type.Repetition.REQUIRED);
    }

    /**
     * Returns a {@link GroupBuilder} to build an optional sub-group.
     *
     * @return a group builder that will return this builder for additional
     *          fields.
     */
    public GroupBuilder<THIS> optionalGroup() {
      return new GroupBuilder<THIS>(self())
          .repetition(Type.Repetition.OPTIONAL);
    }

    /**
     * Returns a {@link GroupBuilder} to build a repeated sub-group.
     *
     * @return a group builder that will return this builder for additional
     *          fields.
     */
    public GroupBuilder<THIS> repeatedGroup() {
      return new GroupBuilder<THIS>(self())
          .repetition(Type.Repetition.REPEATED);
    }

    /**
     * Adds {@code type} as a sub-field to the group configured by this builder.
     *
     * @param type the type to add as a field
     * @return this builder for additional fields.
     */
    public THIS addField(Type type) {
      fields.add(type);
      return self();
    }

    /**
     * Adds {@code types} as sub-fields of the group configured by this builder.
     *
     * @param types an array of types to add as fields
     * @return this builder for additional fields.
     */
    public THIS addFields(Type... types) {
      Collections.addAll(fields, types);
      return self();
    }

    @Override
    protected GroupType build(String name) {
      if (newLogicalTypeSet) {
        return new GroupType(repetition, name, logicalTypeAnnotation, fields, id);
      } else {
        return new GroupType(repetition, name, getOriginalType(), fields, id);
      }
    }

    public MapBuilder<THIS> map(
        Type.Repetition repetition) {
      return new MapBuilder<THIS>(self()).repetition(repetition);
    }

    public MapBuilder<THIS> requiredMap() {
      return new MapBuilder<THIS>(self())
          .repetition(Type.Repetition.REQUIRED);
    }

    public MapBuilder<THIS> optionalMap() {
      return new MapBuilder<THIS>(self())
          .repetition(Type.Repetition.OPTIONAL);
    }

    public ListBuilder<THIS> list(Type.Repetition repetition) {
      return new ListBuilder<THIS>(self()).repetition(repetition);
    }

    public ListBuilder<THIS> requiredList() {
      return list(Type.Repetition.REQUIRED);
    }

    public ListBuilder<THIS> optionalList() {
      return list(Type.Repetition.OPTIONAL);
    }
  }

  /**
   * A builder for {@link GroupType} objects.
   *
   * @param
 The type that this builder will return from
   *          {@link #named(String)} when the type is built.
   */
  public static class GroupBuilder
 extends BaseGroupBuilder {

    private GroupBuilder(P parent) {
      super(parent);
    }

    private GroupBuilder(Class
 returnType) {
      super(returnType);
    }

    @Override
    protected GroupBuilder
 self() {
      return this;
    }
  }

  public abstract static class BaseMapBuilder
      extends Builder<THIS, P> {
    private static final Type STRING_KEY = Types
        .required(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("key");

    public static class KeyBuilder<MP, M extends BaseMapBuilder<MP, M>> extends
        BasePrimitiveBuilder<MP, KeyBuilder<MP, M>> {
      private final M mapBuilder;

      public KeyBuilder(M mapBuilder, PrimitiveTypeName type) {
        super(mapBuilder.parent, type);
        this.mapBuilder = mapBuilder;
        repetition(Type.Repetition.REQUIRED);
      }

      public ValueBuilder<MP, M> value(PrimitiveTypeName type,
                                       Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new ValueBuilder<MP, M>(mapBuilder, type).repetition(repetition);
      }

      public ValueBuilder<MP, M> requiredValue(PrimitiveTypeName type) {
        return value(type, Type.Repetition.REQUIRED);
      }

      public ValueBuilder<MP, M> optionalValue(PrimitiveTypeName type) {
        return value(type, Type.Repetition.OPTIONAL);
      }

      public GroupValueBuilder<MP, M> groupValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new GroupValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public GroupValueBuilder<MP, M> requiredGroupValue() {
        return groupValue(Type.Repetition.REQUIRED);
      }

      public GroupValueBuilder<MP, M> optionalGroupValue() {
        return groupValue(Type.Repetition.OPTIONAL);
      }

      public MapValueBuilder<MP, M> mapValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new MapValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public MapValueBuilder<MP, M> requiredMapValue() {
        return mapValue(Type.Repetition.REQUIRED);
      }

      public MapValueBuilder<MP, M> optionalMapValue() {
        return mapValue(Type.Repetition.OPTIONAL);
      }

      public ListValueBuilder<MP, M> listValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new ListValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public ListValueBuilder<MP, M> requiredListValue() {
        return listValue(Type.Repetition.REQUIRED);
      }

      public ListValueBuilder<MP, M> optionalListValue() {
        return listValue(Type.Repetition.OPTIONAL);
      }

      public M value(Type type) {
        mapBuilder.setKeyType(build("key"));
        mapBuilder.setValueType(type);
        return this.mapBuilder;
      }

      @Override
      public MP named(String name) {
        mapBuilder.setKeyType(build("key"));
        return mapBuilder.named(name);
      }

      @Override
      protected KeyBuilder<MP, M> self() {
        return this;
      }
    }

    public static class ValueBuilder<MP, M extends BaseMapBuilder<MP, M>>
        extends BasePrimitiveBuilder<MP, ValueBuilder<MP, M>> {
      private final M mapBuilder;

      public ValueBuilder(M mapBuilder, PrimitiveTypeName type) {
        super(mapBuilder.parent, type);
        this.mapBuilder = mapBuilder;
      }

      public MP named(String name) {
        mapBuilder.setValueType(build("value"));
        return mapBuilder.named(name);
      }

      @Override
      protected ValueBuilder<MP, M> self() {
        return this;
      }
    }

    public static class GroupKeyBuilder<MP, M extends BaseMapBuilder<MP, M>>
        extends BaseGroupBuilder<MP, GroupKeyBuilder<MP, M>> {
      private final M mapBuilder;

      public GroupKeyBuilder(M mapBuilder) {
        super(mapBuilder.parent);
        this.mapBuilder = mapBuilder;
        repetition(Type.Repetition.REQUIRED);
      }

      @Override
      protected GroupKeyBuilder<MP, M> self() {
        return this;
      }

      public ValueBuilder<MP, M> value(PrimitiveTypeName type,
                                       Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new ValueBuilder<MP, M>(mapBuilder, type).repetition(repetition);
      }

      public ValueBuilder<MP, M> requiredValue(PrimitiveTypeName type) {
        return value(type, Type.Repetition.REQUIRED);
      }

      public ValueBuilder<MP, M> optionalValue(PrimitiveTypeName type) {
        return value(type, Type.Repetition.OPTIONAL);
      }

      public GroupValueBuilder<MP, M> groupValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new GroupValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public GroupValueBuilder<MP, M> requiredGroupValue() {
        return groupValue(Type.Repetition.REQUIRED);
      }

      public GroupValueBuilder<MP, M> optionalGroupValue() {
        return groupValue(Type.Repetition.OPTIONAL);
      }

      public MapValueBuilder<MP, M> mapValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new MapValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public MapValueBuilder<MP, M> requiredMapValue() {
        return mapValue(Type.Repetition.REQUIRED);
      }

      public MapValueBuilder<MP, M> optionalMapValue() {
        return mapValue(Type.Repetition.OPTIONAL);
      }

      public ListValueBuilder<MP, M> listValue(Type.Repetition repetition) {
        mapBuilder.setKeyType(build("key"));
        return new ListValueBuilder<MP, M>(mapBuilder).repetition(repetition);
      }

      public ListValueBuilder<MP, M> requiredListValue() {
        return listValue(Type.Repetition.REQUIRED);
      }

      public ListValueBuilder<MP, M> optionalListValue() {
        return listValue(Type.Repetition.OPTIONAL);
      }

      public M value(Type type) {
        mapBuilder.setKeyType(build("key"));
        mapBuilder.setValueType(type);
        return this.mapBuilder;
      }

      @Override
      public MP named(String name) {
        mapBuilder.setKeyType(build("key"));
        return mapBuilder.named(name);
      }
    }

    public static class GroupValueBuilder<MP, M extends BaseMapBuilder<MP, M>>
        extends BaseGroupBuilder<MP, GroupValueBuilder<MP, M>> {
      private final M mapBuilder;

      public GroupValueBuilder(M mapBuilder) {
        super(mapBuilder.parent);
        this.mapBuilder = mapBuilder;
      }

      public MP named(String name) {
        mapBuilder.setValueType(build("value"));
        return mapBuilder.named(name);
      }

      @Override
      protected GroupValueBuilder<MP, M> self() {
        return this;
      }
    }

    public static class MapValueBuilder<MP, M extends BaseMapBuilder<MP, M>>
        extends BaseMapBuilder<MP, MapValueBuilder<MP, M>> {
      private final M mapBuilder;

      public MapValueBuilder(M mapBuilder) {
        super(mapBuilder.parent);
        this.mapBuilder = mapBuilder;
      }

      public MP named(String name) {
        mapBuilder.setValueType(build("value"));
        return mapBuilder.named(name);
      }

      @Override
      protected MapValueBuilder<MP, M> self() {
        return this;
      }
    }

    public static class ListValueBuilder<MP, M extends BaseMapBuilder<MP, M>>
        extends BaseListBuilder<MP, ListValueBuilder<MP, M>> {
      private final M mapBuilder;

      public ListValueBuilder(M mapBuilder) {
        super(mapBuilder.parent);
        this.mapBuilder = mapBuilder;
      }

      public MP named(String name) {
        mapBuilder.setValueType(build("value"));
        return mapBuilder.named(name);
      }

      @Override
      protected ListValueBuilder<MP, M> self() {
        return this;
      }
    }

    protected void setKeyType(Type keyType) {
      Preconditions.checkState(this.keyType == null,
          "Only one key type can be built with a MapBuilder");
      this.keyType = keyType;
    }

    protected void setValueType(Type valueType) {
      Preconditions.checkState(this.valueType == null,
          "Only one key type can be built with a ValueBuilder");
      this.valueType = valueType;
    }

    private Type keyType = null;
    private Type valueType = null;

    public BaseMapBuilder(P parent) {
      super(parent);
    }

    private BaseMapBuilder(Class
 returnType) {
      super(returnType);
    }

    @Override
    protected abstract THIS self();

    public KeyBuilder
 key(PrimitiveTypeName type) {
      return new KeyBuilder
(self(), type);
    }

    public THIS key(Type type) {
      setKeyType(type);
      return self();
    }

    public GroupKeyBuilder
 groupKey() {
      return new GroupKeyBuilder
(self());
    }

    public ValueBuilder
 value(PrimitiveTypeName type,
                                       Type.Repetition repetition) {
      return new ValueBuilder
(self(), type).repetition(repetition);
    }

    public ValueBuilder
 requiredValue(PrimitiveTypeName type) {
      return value(type, Type.Repetition.REQUIRED);
    }

    public ValueBuilder
 optionalValue(PrimitiveTypeName type) {
      return value(type, Type.Repetition.OPTIONAL);
    }

    public GroupValueBuilder
 groupValue(Type.Repetition repetition) {
      return new GroupValueBuilder
(self()).repetition(repetition);
    }

    public GroupValueBuilder
 requiredGroupValue() {
      return groupValue(Type.Repetition.REQUIRED);
    }

    public GroupValueBuilder
 optionalGroupValue() {
      return groupValue(Type.Repetition.OPTIONAL);
    }

    public MapValueBuilder
 mapValue(Type.Repetition repetition) {
      return new MapValueBuilder
(self()).repetition(repetition);
    }

    public MapValueBuilder
 requiredMapValue() {
      return mapValue(Type.Repetition.REQUIRED);
    }

    public MapValueBuilder
 optionalMapValue() {
      return mapValue(Type.Repetition.OPTIONAL);
    }

    public ListValueBuilder
 listValue(Type.Repetition repetition) {
      return new ListValueBuilder
(self()).repetition(repetition);
    }

    public ListValueBuilder
 requiredListValue() {
      return listValue(Type.Repetition.REQUIRED);
    }

    public ListValueBuilder
 optionalListValue() {
      return listValue(Type.Repetition.OPTIONAL);
    }

    public THIS value(Type type) {
      setValueType(type);
      return self();
    }

    @Override
    protected Type build(String name) {
      Preconditions.checkState(logicalTypeAnnotation == null,
          "MAP is already a logical type and can't be changed.");
      if (keyType == null) {
        keyType = STRING_KEY;
      }

      GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.MAP);
      if (id != null) {
        builder.id(id.intValue());
      }

      if (valueType != null) {
        return builder
            .repeatedGroup().addFields(keyType, valueType).named("map")
            .named(name);
      } else {
        return builder
            .repeatedGroup().addFields(keyType).named("map")
            .named(name);
      }
    }
  }

  public static class MapBuilder
 extends BaseMapBuilder {
    public MapBuilder(P parent) {
      super(parent);
    }

    private MapBuilder(Class
 returnType) {
      super(returnType);
    }

    @Override
    protected MapBuilder
 self() {
      return this;
    }
  }

  public abstract static class BaseListBuilder
      extends Builder<THIS, P> {
    private Type elementType = null;
    private P parent;

    public BaseListBuilder(P parent) {
      super(parent);
      this.parent = parent;
    }

    public BaseListBuilder(Class
 returnType) {
      super(returnType);
    }

    public THIS setElementType(Type elementType) {
      Preconditions.checkState(this.elementType == null,
          "Only one element can be built with a ListBuilder");
      this.elementType = elementType;
      return self();
    }

    public static class ElementBuilder<LP, L extends BaseListBuilder<LP, L>>
        extends BasePrimitiveBuilder<LP, ElementBuilder<LP, L>> {
      private final BaseListBuilder<LP, L> listBuilder;

      public ElementBuilder(L listBuilder, PrimitiveTypeName type) {
        super(((BaseListBuilder<LP, L>) listBuilder).parent, type);
        this.listBuilder = listBuilder;
      }

      public LP named(String name) {
        listBuilder.setElementType(build("element"));
        return listBuilder.named(name);
      }

      @Override
      protected ElementBuilder<LP, L> self() {
        return this;
      }
    }

    public static class GroupElementBuilder<LP, L extends BaseListBuilder<LP, L>>
        extends BaseGroupBuilder<LP, GroupElementBuilder<LP, L>> {
      private final L listBuilder;

      public GroupElementBuilder(L listBuilder) {
        super(((BaseListBuilder<LP, L>) listBuilder).parent);
        this.listBuilder = listBuilder;
      }

      public LP named(String name) {
        listBuilder.setElementType(build("element"));
        return listBuilder.named(name);
      }

      @Override
      protected GroupElementBuilder<LP, L> self() {
        return this;
      }
    }

    public static class MapElementBuilder<LP, L extends BaseListBuilder<LP, L>>
        extends BaseMapBuilder<LP, MapElementBuilder<LP, L>> {

      private final L listBuilder;

      public MapElementBuilder(L listBuilder) {
        super(((BaseListBuilder<LP, L>) listBuilder).parent);
        this.listBuilder = listBuilder;
      }

      @Override
      protected MapElementBuilder<LP, L> self() {
        return this;
      }

      @Override
      public LP named(String name) {
        listBuilder.setElementType(build("element"));
        return listBuilder.named(name);
      }
    }

    public static class ListElementBuilder<LP, L extends BaseListBuilder<LP, L>>
        extends BaseListBuilder<LP, ListElementBuilder<LP, L>> {

      private final L listBuilder;

      public ListElementBuilder(L listBuilder) {
        super(((BaseListBuilder<LP, L>) listBuilder).parent);
        this.listBuilder = listBuilder;
      }

      @Override
      protected ListElementBuilder<LP, L> self() {
        return this;
      }

      @Override
      public LP named(String name) {
        listBuilder.setElementType(build("element"));
        return listBuilder.named(name);
      }
    }

    @Override
    protected abstract THIS self();

    @Override
    protected Type build(String name) {
      Preconditions.checkState(logicalTypeAnnotation == null,
          "LIST is already the logical type and can't be changed");
      Preconditions.checkNotNull(elementType, "List element type");

      GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.LIST);
      if (id != null) {
        builder.id(id.intValue());
      }

      return builder
          .repeatedGroup().addFields(elementType).named("list")
          .named(name);
    }

    public ElementBuilder
 element(PrimitiveTypeName type,
                                           Type.Repetition repetition) {
      return new ElementBuilder
(self(), type).repetition(repetition);
    }

    public ElementBuilder
 requiredElement(PrimitiveTypeName type) {
      return element(type, Type.Repetition.REQUIRED);
    }

    public ElementBuilder
 optionalElement(PrimitiveTypeName type) {
      return element(type, Type.Repetition.OPTIONAL);
    }

    public GroupElementBuilder
 groupElement(Type.Repetition repetition) {
      return new GroupElementBuilder
(self()).repetition(repetition);
    }

    public GroupElementBuilder
 requiredGroupElement() {
      return groupElement(Type.Repetition.REQUIRED);
    }

    public GroupElementBuilder
 optionalGroupElement() {
      return groupElement(Type.Repetition.OPTIONAL);
    }

    public MapElementBuilder
 mapElement(Type.Repetition repetition) {
      return new MapElementBuilder
(self()).repetition(repetition);
    }

    public MapElementBuilder
 requiredMapElement() {
      return mapElement(Type.Repetition.REQUIRED);
    }

    public MapElementBuilder
 optionalMapElement() {
      return mapElement(Type.Repetition.OPTIONAL);
    }

    public ListElementBuilder
 listElement(Type.Repetition repetition) {
      return new ListElementBuilder
(self()).repetition(repetition);
    }

    public ListElementBuilder
 requiredListElement() {
      return listElement(Type.Repetition.REQUIRED);
    }

    public ListElementBuilder
 optionalListElement() {
      return listElement(Type.Repetition.OPTIONAL);
    }

    public BaseListBuilder
 element(Type type) {
      setElementType(type);
      return self();
    }
  }

  public static class ListBuilder
 extends BaseListBuilder {
    public ListBuilder(P parent) {
      super(parent);
    }

    public ListBuilder(Class
 returnType) {
      super(returnType);
    }

    @Override
    protected ListBuilder
 self() {
      return this;
    }
  }

  public static class MessageTypeBuilder extends GroupBuilder<MessageType> {
    private MessageTypeBuilder() {
      super(MessageType.class);
      repetition(Type.Repetition.REQUIRED);
    }

    /**
     * Builds and returns the {@link MessageType} configured by this builder.
     *

     * Note: All primitive types and sub-groups should be added before
     * calling this method.
     *
     * @param name a name for the constructed type
     * @return the final {@code MessageType} configured by this builder.
     */
    @Override
    public MessageType named(String name) {
      Preconditions.checkNotNull(name, "Name is required");
      return new MessageType(name, fields);
    }
  }

  /**
   * Returns a builder to construct a {@link MessageType}.
   *
   * @return a {@link MessageTypeBuilder}
   */
  public static MessageTypeBuilder buildMessage() {
    return new MessageTypeBuilder();
  }

  public static PrimitiveBuilder<PrimitiveType> primitive(PrimitiveTypeName type,
                                                          Type.Repetition repetition) {
    return new PrimitiveBuilder<PrimitiveType>(PrimitiveType.class, type)
        .repetition(repetition);
  }

  /**
   * Returns a builder to construct a required {@link PrimitiveType}.
   *
   * @param type a {@link PrimitiveTypeName} for the constructed type
   * @return a {@link PrimitiveBuilder}
   */
  public static PrimitiveBuilder<PrimitiveType> required(PrimitiveTypeName type) {
    return new PrimitiveBuilder<PrimitiveType>(PrimitiveType.class, type)
        .repetition(Type.Repetition.REQUIRED);
  }

  /**
   * Returns a builder to construct an optional {@link PrimitiveType}.
   *
   * @param type a {@link PrimitiveTypeName} for the constructed type
   * @return a {@link PrimitiveBuilder}
   */
  public static PrimitiveBuilder<PrimitiveType> optional(PrimitiveTypeName type) {
    return new PrimitiveBuilder<PrimitiveType>(PrimitiveType.class, type)
        .repetition(Type.Repetition.OPTIONAL);
  }

  /**
   * Returns a builder to construct a repeated {@link PrimitiveType}.
   *
   * @param type a {@link PrimitiveTypeName} for the constructed type
   * @return a {@link PrimitiveBuilder}
   */
  public static PrimitiveBuilder<PrimitiveType> repeated(PrimitiveTypeName type) {
    return new PrimitiveBuilder<PrimitiveType>(PrimitiveType.class, type)
        .repetition(Type.Repetition.REPEATED);
  }

  public static GroupBuilder<GroupType> buildGroup(
      Type.Repetition repetition) {
    return new GroupBuilder<GroupType>(GroupType.class).repetition(repetition);
  }

  /**
   * Returns a builder to construct a required {@link GroupType}.
   *
   * @return a {@link GroupBuilder}
   */
  public static GroupBuilder<GroupType> requiredGroup() {
    return new GroupBuilder<GroupType>(GroupType.class)
        .repetition(Type.Repetition.REQUIRED);
  }

  /**
   * Returns a builder to construct an optional {@link GroupType}.
   *
   * @return a {@link GroupBuilder}
   */
  public static GroupBuilder<GroupType> optionalGroup() {
    return new GroupBuilder<GroupType>(GroupType.class)
        .repetition(Type.Repetition.OPTIONAL);
  }

  /**
   * Returns a builder to construct a repeated {@link GroupType}.
   *
   * @return a {@link GroupBuilder}
   */
  public static GroupBuilder<GroupType> repeatedGroup() {
    return new GroupBuilder<GroupType>(GroupType.class)
        .repetition(Type.Repetition.REPEATED);
  }

  public static MapBuilder<GroupType> map(Type.Repetition repetition) {
    return new MapBuilder<GroupType>(GroupType.class).repetition(repetition);
  }

  public static MapBuilder<GroupType> requiredMap() {
    return map(Type.Repetition.REQUIRED);
  }

  public static MapBuilder<GroupType> optionalMap() {
    return map(Type.Repetition.OPTIONAL);
  }

  public static ListBuilder<GroupType> list(Type.Repetition repetition) {
    return new ListBuilder<GroupType>(GroupType.class).repetition(repetition);
  }

  public static ListBuilder<GroupType> requiredList() {
    return list(Type.Repetition.REQUIRED);
  }

  public static ListBuilder<GroupType> optionalList() {
    return list(Type.Repetition.OPTIONAL);
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.dictionary;

import java.util.ArrayList;
import java.util.List;

/**
 * An append-only integer list
 * avoids autoboxing and buffer resizing
 */
public class IntList {

  static final int MAX_SLAB_SIZE = 64 * 1024;
  static final int INITIAL_SLAB_SIZE = 4 * 1024;

  // Double slab size till we reach the max slab size. At that point we just add slabs of size
  // MAX_SLAB_SIZE. This ensures we don't allocate very large slabs from the start if we don't have
  // too much data.
  private int currentSlabSize = INITIAL_SLAB_SIZE;

  /**
   * Visible for testing to verify the current slab size
   */
  int getCurrentSlabSize() {
    return currentSlabSize;
  }

  /**
   * to iterate on the content of the list
   * not an actual iterator to avoid autoboxing
   */
  public static class IntIterator {

    private final int[][] slabs;
    private final int count;

    private int current;
    private int currentRow;
    private int currentCol;

    /**
     * slabs will be iterated in order up to the provided count
     * as the last slab may not be full
     * @param slabs contain the ints
     * @param count count of ints
     */
    public IntIterator(int[][] slabs, int count) {
      this.slabs = slabs;
      this.count = count;
    }

    /**
     * @return whether there is a next value
     */
    public boolean hasNext() {
      return current < count;
    }

    /**
     * @return the next int
     */
    public int next() {
      final int result = slabs[currentRow][currentCol];
      incrementPosition();
      return result;
    }

    private void incrementPosition() {
      current++;
      currentCol++;
      if (currentCol >= slabs[currentRow].length) {
        currentCol = 0;
        currentRow++;
      }
    }
  }

  private List<int[]> slabs = new ArrayList<int[]>();

  // Lazy initialize currentSlab only when needed to save on memory in cases where items might
  // not be added
  private int[] currentSlab;
  private int currentSlabPos;

  private void allocateSlab() {
    currentSlab = new int[currentSlabSize];
    currentSlabPos = 0;
  }

  // Double slab size up to the MAX_SLAB_SIZE limit
  private void updateCurrentSlabSize() {
    if (currentSlabSize < MAX_SLAB_SIZE) {
      currentSlabSize *= 2;
      if (currentSlabSize > MAX_SLAB_SIZE) {
        currentSlabSize = MAX_SLAB_SIZE;
      }
    }
  }

  /**
   * @param i value to append to the end of the list
   */
  public void add(int i) {
    if (currentSlab == null) {
      allocateSlab();
    } else if (currentSlabPos == currentSlab.length) {
      slabs.add(currentSlab);
      updateCurrentSlabSize();
      allocateSlab();
    }

    currentSlab[currentSlabPos] = i;
    ++ currentSlabPos;
  }

  /**
   * (not an actual Iterable)
   * @return an IntIterator on the content
   */
  public IntIterator iterator() {
    if (currentSlab == null) {
      allocateSlab();
    }

    int[][] itSlabs = slabs.toArray(new int[slabs.size() + 1][]);
    itSlabs[slabs.size()] = currentSlab;
    return new IntIterator(itSlabs, size());
  }

  /**
   * @return the current size of the list
   */
  public int size() {
    int size = currentSlabPos;
    for (int [] slab : slabs) {
      size += slab.length;
    }

    return size;
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet;

import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.parquet.SemanticVersion.SemanticVersionParseException;
import org.apache.parquet.VersionParser.ParsedVersion;
import org.apache.parquet.VersionParser.VersionParseException;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * There was a bug (PARQUET-251) that caused the statistics metadata
 * for binary columns to be corrupted in the write path.
 *
 * This class is used to detect whether a file was written with this bug,
 * and thus it's statistics should be ignored / not trusted.
 */
public class CorruptStatistics {
  private static final AtomicBoolean alreadyLogged = new AtomicBoolean(false);

  private static final Logger LOG = LoggerFactory.getLogger(CorruptStatistics.class);

  // the version in which the bug described by jira: PARQUET-251 was fixed
  // the bug involved writing invalid binary statistics, so stats written prior to this
  // fix must be ignored / assumed invalid
  private static final SemanticVersion PARQUET_251_FIXED_VERSION = new SemanticVersion(1, 8, 0);
  private static final SemanticVersion CDH_5_PARQUET_251_FIXED_START = new SemanticVersion(1, 5, 0, null, "cdh5.5.0", null);
  private static final SemanticVersion CDH_5_PARQUET_251_FIXED_END = new SemanticVersion(1, 5, 0);

  /**
   * Decides if the statistics from a file created by createdBy (the created_by field from parquet format)
   * should be ignored because they are potentially corrupt.
   *
   * @param createdBy the created-by string from a file footer
   * @param columnType the type of the column that this is checking
   * @return true if the statistics may be invalid and should be ignored, false otherwise
   */
  public static boolean shouldIgnoreStatistics(String createdBy, PrimitiveTypeName columnType) {

    if (columnType != PrimitiveTypeName.BINARY && columnType != PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      // the bug only applies to binary columns
      return false;
    }

    if (Strings.isNullOrEmpty(createdBy)) {
      // created_by is not populated, which could have been caused by
      // parquet-mr during the same time as PARQUET-251, see PARQUET-297
      warnOnce("Ignoring statistics because created_by is null or empty! See PARQUET-251 and PARQUET-297");
      return true;
    }

    try {
      ParsedVersion version = VersionParser.parse(createdBy);

      if (!"parquet-mr".equals(version.application)) {
        // assume other applications don't have this bug
        return false;
      }

      if (Strings.isNullOrEmpty(version.version)) {
        warnOnce("Ignoring statistics because created_by did not contain a semver (see PARQUET-251): " + createdBy);
        return true;
      }

      SemanticVersion semver = SemanticVersion.parse(version.version);

      if (semver.compareTo(PARQUET_251_FIXED_VERSION) < 0 &&
          !(semver.compareTo(CDH_5_PARQUET_251_FIXED_START) >= 0 &&
              semver.compareTo(CDH_5_PARQUET_251_FIXED_END) < 0)) {
        warnOnce("Ignoring statistics because this file was created prior to "
            + PARQUET_251_FIXED_VERSION
            + ", see PARQUET-251");
        return true;
      }

      // this file was created after the fix
      return false;
    } catch (RuntimeException e) {
      // couldn't parse the created_by field, log what went wrong, don't trust the stats,
      // but don't make this fatal.
      warnParseErrorOnce(createdBy, e);
      return true;
    } catch (SemanticVersionParseException e) {
      // couldn't parse the created_by field, log what went wrong, don't trust the stats,
      // but don't make this fatal.
      warnParseErrorOnce(createdBy, e);
      return true;
    } catch (VersionParseException e) {
      // couldn't parse the created_by field, log what went wrong, don't trust the stats,
      // but don't make this fatal.
      warnParseErrorOnce(createdBy, e);
      return true;
    }
  }

  private static void warnParseErrorOnce(String createdBy, Throwable e) {
    if(!alreadyLogged.getAndSet(true)) {
      LOG.warn("Ignoring statistics because created_by could not be parsed (see PARQUET-251): " + createdBy, e);
    }
  }

  private static void warnOnce(String message) {
    if(!alreadyLogged.getAndSet(true)) {
      LOG.warn(message);
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.io;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.column.impl.ColumnReadStoreImpl;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;
import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * used to read reassembled records
 *
 * @param <T> the type of the materialized record
 */
class RecordReaderImplementation<T> extends RecordReader<T> {
  private static final Logger LOG = LoggerFactory.getLogger(RecordReaderImplementation.class);

  public static class Case {

    private int id;
    private final int startLevel;
    private final int depth;
    private final int nextLevel;
    private final boolean goingUp;
    private final boolean goingDown;
    private final int nextState;
    private final boolean defined;

    public Case(int startLevel, int depth, int nextLevel, int nextState, boolean defined) {
      this.startLevel = startLevel;
      this.depth = depth;
      this.nextLevel = nextLevel;
      this.nextState = nextState;
      this.defined = defined;
      // means going up the tree (towards the leaves) of the record
      // true if we need to open up groups in this case
      goingUp = startLevel <= depth;
      // means going down the tree (towards the root) of the record
      // true if we need to close groups in this case
      goingDown = depth + 1 > nextLevel;
    }

    public void setID(int id) {
      this.id = id;
    }

    @Override
    // this implementation is buggy but the simpler one bellow has duplicates.
    // it still works but generates more code than necessary
    // a middle ground is necessary
//    public int hashCode() {
//      int hashCode = 0;
//      if (goingUp) {
//        hashCode += 1 * (1 + startLevel) + 2 * (1 + depth);
//      }
//      if (goingDown) {
//        hashCode += 3 * (1 + depth) + 5 * (1 + nextLevel);
//      }
//      return hashCode;
//    }

    public int hashCode() {
      int hashCode = 17;
      hashCode += 31 * startLevel;
      hashCode += 31 * depth;
      hashCode += 31 * nextLevel;
      hashCode += 31 * nextState;
      hashCode += 31 * (defined ? 0 : 1);
      return hashCode;
    }

    @Override
    public boolean equals(Object obj) {
      if (obj instanceof Case) {
        return equals((Case)obj);
      }
      return false;
    };

    // see comment for hashCode above
//    public boolean equals(Case other) {
//      if (goingUp && !other.goingUp || !goingUp && other.goingUp) {
//        return false;
//      }
//      if (goingUp && other.goingUp && (startLevel != other.startLevel || depth != other.depth)) {
//        return false;
//      }
//      if (goingDown && !other.goingDown || !goingDown && other.goingDown) {
//        return false;
//      }
//      if (goingDown && other.goingDown && (depth != other.depth || nextLevel != other.nextLevel)) {
//        return false;
//      }
//      return true;
//    }

    public boolean equals(Case other) {
      return other != null
          && startLevel == other.startLevel
          && depth == other.depth
          && nextLevel == other.nextLevel
          && nextState == other.nextState
          && ((defined && other.defined) || (!defined && !other.defined));
    }

    public int getID() {
      return id;
    }

    public int getStartLevel() {
      return startLevel;
    }

    public int getDepth() {
      return depth;
    }
    public int getNextLevel() {
      return nextLevel;
    }

    public int getNextState() {
      return nextState;
    }

    public boolean isGoingUp() {
      return goingUp;
    }

    public boolean isGoingDown() {
      return goingDown;
    }

    public boolean isDefined() {
      return defined;
    }

    @Override
    public String toString() {
      return "Case " + startLevel + " -> " + depth + " -> " + nextLevel + "; goto sate_"+getNextState();
    }

  }

  public static class State {

    public final int id;
    public final PrimitiveColumnIO primitiveColumnIO;
    public final int maxDefinitionLevel;
    public final int maxRepetitionLevel;
    public final PrimitiveTypeName primitive;
    public final ColumnReader column;
    public final String[] fieldPath; // indexed by currentLevel
    public final int[] indexFieldPath; // indexed by currentLevel
    public final GroupConverter[] groupConverterPath;
    public final PrimitiveConverter primitiveConverter;
    public final String primitiveField;
    public final int primitiveFieldIndex;
    public final int[] nextLevel; //indexed by next r

    private int[] definitionLevelToDepth; // indexed by current d
    private State[] nextState; // indexed by next r
    private Case[][][] caseLookup;
    private List<Case> definedCases;
    private List<Case> undefinedCases;

    private State(int id, PrimitiveColumnIO primitiveColumnIO, ColumnReader column, int[] nextLevel, GroupConverter[] groupConverterPath, PrimitiveConverter primitiveConverter) {
      this.id = id;
      this.primitiveColumnIO = primitiveColumnIO;
      this.maxDefinitionLevel = primitiveColumnIO.getDefinitionLevel();
      this.maxRepetitionLevel = primitiveColumnIO.getRepetitionLevel();
      this.column = column;
      this.nextLevel = nextLevel;
      this.groupConverterPath = groupConverterPath;
      this.primitiveConverter = primitiveConverter;
      this.primitive = primitiveColumnIO.getType().asPrimitiveType().getPrimitiveTypeName();
      this.fieldPath = primitiveColumnIO.getFieldPath();
      this.primitiveField = fieldPath[fieldPath.length - 1];
      this.indexFieldPath = primitiveColumnIO.getIndexFieldPath();
      this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1];
    }

    public int getDepth(int definitionLevel) {
      return definitionLevelToDepth[definitionLevel];
    }

    public List<Case> getDefinedCases() {
      return definedCases;
    }

    public List<Case> getUndefinedCases() {
      return undefinedCases;
    }

    public Case getCase(int currentLevel, int d, int nextR) {
      return caseLookup[currentLevel][d][nextR];
    }

    public State getNextState(int nextR) {
      return nextState[nextR];
    }
  }

  private final GroupConverter recordRootConverter;
  private final RecordMaterializer<T> recordMaterializer;

  private State[] states;
  private ColumnReader[] columnReaders;

  private boolean shouldSkipCurrentRecord = false;

  /**
   * @param root the root of the schema
   * @param recordMaterializer responsible of materializing the records
   * @param validating whether we should validate against the schema
   * @param columnStore where to read the column data from
   */
  public RecordReaderImplementation(MessageColumnIO root, RecordMaterializer<T> recordMaterializer, boolean validating, ColumnReadStoreImpl columnStore) {
    this.recordMaterializer = recordMaterializer;
    this.recordRootConverter = recordMaterializer.getRootConverter(); // TODO: validator(wrap(recordMaterializer), validating, root.getType());
    PrimitiveColumnIO[] leaves = root.getLeaves().toArray(new PrimitiveColumnIO[root.getLeaves().size()]);
    columnReaders = new ColumnReader[leaves.length];
    int[][] nextColumnIdxForRepLevel = new int[leaves.length][];
    int[][] levelToClose = new int[leaves.length][];
    GroupConverter[][] groupConverterPaths = new GroupConverter[leaves.length][];
    PrimitiveConverter[] leafConverters = new PrimitiveConverter[leaves.length];
    int[] firstIndexForLevel  = new int[256]; // "256 levels of nesting ought to be enough for anybody"
    // build the automaton
    for (int i = 0; i < leaves.length; i++) {
      PrimitiveColumnIO leafColumnIO = leaves[i];
      //generate converters along the path from root to leaf
      final int[] indexFieldPath = leafColumnIO.getIndexFieldPath();
      groupConverterPaths[i] = new GroupConverter[indexFieldPath.length - 1];
      GroupConverter current = this.recordRootConverter;
      for (int j = 0; j < indexFieldPath.length - 1; j++) {
        current = current.getConverter(indexFieldPath[j]).asGroupConverter();
        groupConverterPaths[i][j] = current;
      }
      leafConverters[i] = current.getConverter(indexFieldPath[indexFieldPath.length - 1]).asPrimitiveConverter();
      columnReaders[i] = columnStore.getColumnReader(leafColumnIO.getColumnDescriptor());
      int maxRepetitionLevel = leafColumnIO.getRepetitionLevel();
      nextColumnIdxForRepLevel[i] = new int[maxRepetitionLevel+1];

      levelToClose[i] = new int[maxRepetitionLevel+1]; //next level
      for (int nextRepLevel = 0; nextRepLevel <= maxRepetitionLevel; ++nextRepLevel) {
        // remember which is the first for this level
        if (leafColumnIO.isFirst(nextRepLevel)) {
          firstIndexForLevel[nextRepLevel] = i;
        }
        int nextColIdx;
        //TODO: when we use nextColumnIdxForRepLevel, should we provide current rep level or the rep level for next item
        // figure out automaton transition
        if (nextRepLevel == 0) { // 0 always means jump to the next (the last one being a special case)
          nextColIdx = i + 1;
        } else if (leafColumnIO.isLast(nextRepLevel)) { // when we are at the last of the next repetition level we jump back to the first
          nextColIdx = firstIndexForLevel[nextRepLevel];
        } else { // otherwise we just go back to the next.
          nextColIdx = i + 1;
        }
        // figure out which level down the tree we need to go back
        if (nextColIdx == leaves.length) { // reached the end of the record => close all levels
          levelToClose[i][nextRepLevel] = 0;
        } else if (leafColumnIO.isLast(nextRepLevel)) { // reached the end of this level => close the repetition level
          ColumnIO parent = leafColumnIO.getParent(nextRepLevel);
          levelToClose[i][nextRepLevel] = parent.getFieldPath().length - 1;
        } else { // otherwise close until the next common parent
          levelToClose[i][nextRepLevel] = getCommonParentLevel(
              leafColumnIO.getFieldPath(),
              leaves[nextColIdx].getFieldPath());
        }
        // sanity check: that would be a bug
        if (levelToClose[i][nextRepLevel] > leaves[i].getFieldPath().length-1) {
          throw new ParquetEncodingException(Arrays.toString(leaves[i].getFieldPath())+" -("+nextRepLevel+")-> "+levelToClose[i][nextRepLevel]);
        }
        nextColumnIdxForRepLevel[i][nextRepLevel] = nextColIdx;
      }
    }
    states = new State[leaves.length];
    for (int i = 0; i < leaves.length; i++) {
      states[i] = new State(i, leaves[i], columnReaders[i], levelToClose[i], groupConverterPaths[i], leafConverters[i]);

      int[] definitionLevelToDepth = new int[states[i].primitiveColumnIO.getDefinitionLevel() + 1];
      // for each possible definition level, determine the depth at which to create groups
      final ColumnIO[] path = states[i].primitiveColumnIO.getPath();
      int depth = 0;
      for (int d = 0; d < definitionLevelToDepth.length; ++d) {
        while (depth < (states[i].fieldPath.length - 1)
          && d >= path[depth + 1].getDefinitionLevel()
          ) {
          ++ depth;
        }
        definitionLevelToDepth[d] = depth - 1;
      }
      states[i].definitionLevelToDepth = definitionLevelToDepth;
    }
    for (int i = 0; i < leaves.length; i++) {
      State state = states[i];
      int[] nextStateIds = nextColumnIdxForRepLevel[i];
      state.nextState = new State[nextStateIds.length];
      for (int j = 0; j < nextStateIds.length; j++) {
        state.nextState[j] = nextStateIds[j] == states.length ? null : states[nextStateIds[j]];
      }
    }
    for (int i = 0; i < states.length; i++) {
      State state = states[i];
      final Map<Case, Case> definedCases = new HashMap<Case, Case>();
      final Map<Case, Case> undefinedCases = new HashMap<Case, Case>();
      Case[][][] caseLookup = new Case[state.fieldPath.length][][];
      for (int currentLevel = 0; currentLevel < state.fieldPath.length; ++ currentLevel) {
        caseLookup[currentLevel] = new Case[state.maxDefinitionLevel+1][];
        for (int d = 0; d <= state.maxDefinitionLevel; ++ d) {
          caseLookup[currentLevel][d] = new Case[state.maxRepetitionLevel+1];
          for (int nextR = 0; nextR <= state.maxRepetitionLevel; ++ nextR) {
            int caseStartLevel = currentLevel;
            int caseDepth = Math.max(state.getDepth(d), caseStartLevel - 1);
            int caseNextLevel = Math.min(state.nextLevel[nextR], caseDepth + 1);
            Case currentCase = new Case(caseStartLevel, caseDepth, caseNextLevel, getNextReader(state.id, nextR), d == state.maxDefinitionLevel);
            Map<Case, Case> cases = currentCase.isDefined() ? definedCases : undefinedCases;
            if (!cases.containsKey(currentCase)) {
              currentCase.setID(cases.size());
              cases.put(currentCase, currentCase);
            } else {
              currentCase = cases.get(currentCase);
            }
            caseLookup[currentLevel][d][nextR] = currentCase;
          }
        }
      }
      state.caseLookup = caseLookup;
      state.definedCases = new ArrayList<Case>(definedCases.values());
      state.undefinedCases = new ArrayList<Case>(undefinedCases.values());
      Comparator<Case> caseComparator = new Comparator<Case>() {
        @Override
        public int compare(Case o1, Case o2) {
          return o1.id - o2.id;
        }
      };
      Collections.sort(state.definedCases, caseComparator);
      Collections.sort(state.undefinedCases, caseComparator);
    }
  }

  //TODO: have those wrappers for a converter
  private RecordConsumer validator(RecordConsumer recordConsumer, boolean validating, MessageType schema) {
    return validating ? new ValidatingRecordConsumer(recordConsumer, schema) : recordConsumer;
  }

  private RecordConsumer wrap(RecordConsumer recordConsumer) {
    if (LOG.isDebugEnabled()) {
      return new RecordConsumerLoggingWrapper(recordConsumer);
    }
    return recordConsumer;
  }

  /**
   * @see org.apache.parquet.io.RecordReader#read()
   */
  @Override
  public T read() {
    int currentLevel = 0;
    recordRootConverter.start();
    State currentState = states[0];
    do {
      ColumnReader columnReader = currentState.column;
      int d = columnReader.getCurrentDefinitionLevel();
      // creating needed nested groups until the current field (opening tags)
      int depth = currentState.definitionLevelToDepth[d];
      for (; currentLevel <= depth; ++currentLevel) {
        currentState.groupConverterPath[currentLevel].start();
      }
      // currentLevel = depth + 1 at this point
      // set the current value
      if (d >= currentState.maxDefinitionLevel) {
        // not null
        columnReader.writeCurrentValueToConverter();
      }
      columnReader.consume();

      int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
      // level to go to close current groups
      int next = currentState.nextLevel[nextR];
      for (; currentLevel > next; currentLevel--) {
        currentState.groupConverterPath[currentLevel - 1].end();
      }

      currentState = currentState.nextState[nextR];
    } while (currentState != null);
    recordRootConverter.end();
    T record = recordMaterializer.getCurrentRecord();
    shouldSkipCurrentRecord = record == null;
    if (shouldSkipCurrentRecord) {
      recordMaterializer.skipCurrentRecord();
    }
    return record;
  }

  @Override
  public boolean shouldSkipCurrentRecord() {
    return shouldSkipCurrentRecord;
  }

  private static void log(String string) {
    LOG.debug(string);
  }

  int getNextReader(int current, int nextRepetitionLevel) {
    State nextState = states[current].nextState[nextRepetitionLevel];
    return nextState == null ? states.length : nextState.id;
  }

  int getNextLevel(int current, int nextRepetitionLevel) {
    return states[current].nextLevel[nextRepetitionLevel];
  }

  private int getCommonParentLevel(String[] previous, String[] next) {
    int i = 0;
    while (i < Math.min(previous.length, next.length) && previous[i].equals(next[i])) {
      ++i;
    }
    return i;
  }

  protected int getStateCount() {
    return states.length;
  }

  protected State getState(int i) {
    return states[i];
  }

  protected RecordMaterializer<T> getMaterializer() {
    return recordMaterializer;
  }

  protected Converter getRecordConsumer() {
    return recordRootConverter;
  }

  protected Iterable<ColumnReader> getColumnReaders() {
    // Converting the array to an iterable ensures that the array cannot be altered
    return Arrays.asList(columnReaders);
  }
}