/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.avro;

import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
import it.unimi.dsi.fastutil.floats.FloatArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.LinkedHashMap;
import org.apache.avro.AvroTypeException;
import org.apache.avro.Conversion;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.SchemaCompatibility;
import org.apache.avro.generic.GenericData;
import org.apache.avro.reflect.AvroIgnore;
import org.apache.avro.reflect.AvroName;
import org.apache.avro.reflect.AvroSchema;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.Stringable;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.util.ClassUtils;
import org.apache.parquet.Preconditions;
import org.apache.parquet.avro.AvroConverters.FieldStringConverter;
import org.apache.parquet.avro.AvroConverters.FieldStringableConverter;
import org.apache.parquet.filter2.predicate.SchemaCompatibilityValidator;
import org.apache.parquet.io.InvalidRecordException;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;

import static org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE;
import static org.apache.avro.SchemaCompatibility.checkReaderWriterCompatibility;
import static org.apache.parquet.schema.Type.Repetition.REPEATED;
import static org.apache.parquet.schema.Type.Repetition.REQUIRED;

/**
 * This {@link Converter} class materializes records for a given
 * {@link GenericData Avro data model}. This replaces
 * {@link AvroIndexedRecordConverter} and works with generic, specific, and
 * reflect records.
 *
 * @param <T> a subclass of Avro's IndexedRecord
 */
class AvroRecordConverter<T> extends AvroConverters.AvroGroupConverter {

  private static final String STRINGABLE_PROP = "avro.java.string";
  private static final String JAVA_CLASS_PROP = "java-class";
  private static final String JAVA_KEY_CLASS_PROP = "java-key-class";

  protected T currentRecord = null;
  private ParentValueContainer rootContainer = null;
  private final Converter[] converters;

  private final Schema avroSchema;

  private final GenericData model;
  private final Map<Schema.Field, Object> recordDefaults = new HashMap<Schema.Field, Object>();

  public AvroRecordConverter(MessageType parquetSchema, Schema avroSchema,
                             GenericData baseModel) {
    this(null, parquetSchema, avroSchema, baseModel);
    LogicalType logicalType = avroSchema.getLogicalType();
    Conversion<?> conversion = baseModel.getConversionFor(logicalType);
    this.rootContainer = ParentValueContainer.getConversionContainer(new ParentValueContainer() {
      @Override
      @SuppressWarnings("unchecked")
      public void add(Object value) {
        AvroRecordConverter.this.currentRecord = (T) value;
      }
    }, conversion, avroSchema);
  }

  public AvroRecordConverter(ParentValueContainer parent,
                             GroupType parquetSchema, Schema avroSchema,
                             GenericData model) {
    super(parent);
    this.avroSchema = avroSchema;
    this.model = (model == null ? ReflectData.get() : model);
    this.converters = new Converter[parquetSchema.getFieldCount()];

    Map<String, Integer> avroFieldIndexes = new HashMap<String, Integer>();
    int avroFieldIndex = 0;
    for (Schema.Field field: avroSchema.getFields()) {
        avroFieldIndexes.put(field.name(), avroFieldIndex++);
    }

    Class<?> recordClass = null;
    if (model instanceof ReflectData) {
      recordClass = getDatumClass(avroSchema, model);
    }

    Map<String, Class<?>> fields = getFieldsByName(recordClass, false);

    int parquetFieldIndex = 0;
    for (Type parquetField: parquetSchema.getFields()) {
      final Schema.Field avroField = getAvroField(parquetField.getName());
      Schema nonNullSchema = AvroSchemaConverter.getNonNull(avroField.schema());
      final int finalAvroIndex = avroFieldIndexes.remove(avroField.name());
      ParentValueContainer container = new ParentValueContainer() {
        @Override
        public void add(Object value) {
          AvroRecordConverter.this.set(avroField.name(), finalAvroIndex, value);
        }
      };

      Class<?> fieldClass = fields.get(avroField.name());
      converters[parquetFieldIndex] = newConverter(
          nonNullSchema, parquetField, this.model, fieldClass, container);

      // @Stringable doesn't affect the reflected schema; must be enforced here
      if (recordClass != null &&
          converters[parquetFieldIndex] instanceof FieldStringConverter) {
        try {
          Field field = recordClass.getDeclaredField(avroField.name());
          if (field.isAnnotationPresent(Stringable.class)) {
            converters[parquetFieldIndex] = new FieldStringableConverter(
                container, field.getType());
          }
        } catch (NoSuchFieldException e) {
          // must not be stringable
        }
      }

      parquetFieldIndex += 1;
    }

    // store defaults for any new Avro fields from avroSchema that are not in
    // the writer schema (parquetSchema)
    for (String fieldName : avroFieldIndexes.keySet()) {
      Schema.Field field = avroSchema.getField(fieldName);
      if (field.schema().getType() == Schema.Type.NULL) {
        continue; // skip null since Parquet does not write nulls
      }
      if (field.defaultVal() == null || this.model.getDefaultValue(field) == null) {
        continue; // field has no default
      }
      // use this.model because model may be null
      recordDefaults.put(field, this.model.getDefaultValue(field));
    }
  }

  // this was taken from Avro's ReflectData
  private static Map<String, Class<?>> getFieldsByName(Class<?> recordClass,
                                                       boolean excludeJava) {
    Map<String, Class<?>> fields = new LinkedHashMap<String, Class<?>>();

    if (recordClass != null) {
      Class<?> current = recordClass;
      do {
        if (excludeJava && current.getPackage() != null
            && current.getPackage().getName().startsWith("java.")) {
          break; // skip java built-in classes
        }
        for (Field field : current.getDeclaredFields()) {
          if (field.isAnnotationPresent(AvroIgnore.class) ||
              isTransientOrStatic(field)) {
            continue;
          }
          AvroName altName = field.getAnnotation(AvroName.class);
          Class<?> existing = fields.put(
              altName != null ? altName.value() : field.getName(),
              field.getType());
          if (existing != null) {
            throw new AvroTypeException(
                current + " contains two fields named: " + field.getName());
          }
        }
        current = current.getSuperclass();
      } while (current != null);
    }

    return fields;
  }

  private static boolean isTransientOrStatic(Field field) {
    return (field.getModifiers() & (Modifier.TRANSIENT | Modifier.STATIC)) != 0;
  }

  private Schema.Field getAvroField(String parquetFieldName) {
    Schema.Field avroField = avroSchema.getField(parquetFieldName);
    if (avroField != null) {
      return avroField;
    }

    for (Schema.Field f : avroSchema.getFields()) {
      if (f.aliases().contains(parquetFieldName)) {
        return f;
      }
    }

    throw new InvalidRecordException(String.format(
        "Parquet/Avro schema mismatch: Avro field '%s' not found",
        parquetFieldName));
  }

  private static Converter newConverter(
      Schema schema, Type type, GenericData model, ParentValueContainer setter) {
    return newConverter(schema, type, model, null, setter);
  }

  private static Converter newConverter(Schema schema, Type type,
      GenericData model, Class<?> knownClass, ParentValueContainer setter) {
    LogicalType logicalType = schema.getLogicalType();
    Conversion<?> conversion;
    if (knownClass != null) {
      conversion = model.getConversionByClass(knownClass, logicalType);
    } else {
      conversion = model.getConversionFor(logicalType);
    }

    ParentValueContainer parent = ParentValueContainer
        .getConversionContainer(setter, conversion, schema);

    if (schema.getType().equals(Schema.Type.BOOLEAN)) {
      return new AvroConverters.FieldBooleanConverter(parent);
    } else if (schema.getType().equals(Schema.Type.INT)) {
      Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model);
      if (datumClass == null) {
        return new AvroConverters.FieldIntegerConverter(parent);
      } else if (datumClass == byte.class || datumClass == Byte.class) {
        return new AvroConverters.FieldByteConverter(parent);
      } else if (datumClass == short.class || datumClass == Short.class) {
        return new AvroConverters.FieldShortConverter(parent);
      } else if (datumClass == char.class || datumClass == Character.class) {
        return new AvroConverters.FieldCharConverter(parent);
      }
      return new AvroConverters.FieldIntegerConverter(parent);
    } else if (schema.getType().equals(Schema.Type.LONG)) {
      return new AvroConverters.FieldLongConverter(parent);
    } else if (schema.getType().equals(Schema.Type.FLOAT)) {
      return new AvroConverters.FieldFloatConverter(parent);
    } else if (schema.getType().equals(Schema.Type.DOUBLE)) {
      return new AvroConverters.FieldDoubleConverter(parent);
    } else if (schema.getType().equals(Schema.Type.BYTES)) {
      Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model);
      if (datumClass == null) {
        return new AvroConverters.FieldByteBufferConverter(parent);
      } else if (datumClass.isArray() && datumClass.getComponentType() == byte.class) {
        return new AvroConverters.FieldByteArrayConverter(parent);
      }
      return new AvroConverters.FieldByteBufferConverter(parent);
    } else if (schema.getType().equals(Schema.Type.STRING)) {
      return newStringConverter(schema, model, parent);
    } else if (schema.getType().equals(Schema.Type.RECORD)) {
      return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
    } else if (schema.getType().equals(Schema.Type.ENUM)) {
      return new AvroConverters.FieldEnumConverter(parent, schema, model);
    } else if (schema.getType().equals(Schema.Type.ARRAY)) {
      Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model);
      if (datumClass != null && datumClass.isArray()) {
        return new AvroArrayConverter(
            parent, type.asGroupType(), schema, model, datumClass);
      } else {
        return new AvroCollectionConverter(
            parent, type.asGroupType(), schema, model, datumClass);
      }
    } else if (schema.getType().equals(Schema.Type.MAP)) {
      return new MapConverter(parent, type.asGroupType(), schema, model);
    } else if (schema.getType().equals(Schema.Type.UNION)) {
      return new AvroUnionConverter(parent, type, schema, model);
    } else if (schema.getType().equals(Schema.Type.FIXED)) {
      return new AvroConverters.FieldFixedConverter(parent, schema, model);
    }
    throw new UnsupportedOperationException(String.format(
        "Cannot convert Avro type: %s to Parquet type: %s", schema, type));
  }

  private static Converter newStringConverter(Schema schema, GenericData model,
                                              ParentValueContainer parent) {
    Class<?> stringableClass = getStringableClass(schema, model);
    if (stringableClass == String.class) {
      return new FieldStringConverter(parent);
    } else if (stringableClass == CharSequence.class) {
      return new AvroConverters.FieldUTF8Converter(parent);
    }
    return new FieldStringableConverter(parent, stringableClass);
  }

  private static Class<?> getStringableClass(Schema schema, GenericData model) {
    if (model instanceof SpecificData) {
      // both specific and reflect (and any subclasses) use this logic
      boolean isMap = (schema.getType() == Schema.Type.MAP);
      String stringableClass = schema.getProp(
          isMap ? JAVA_KEY_CLASS_PROP : JAVA_CLASS_PROP);
      if (stringableClass != null) {
        try {
          return ClassUtils.forName(model.getClassLoader(), stringableClass);
        } catch (ClassNotFoundException e) {
          // not available, use a String instead
        }
      }
    }

    if (ReflectData.class.isAssignableFrom(model.getClass())) {
      // reflect uses String, not Utf8
      return String.class;
    }

    // generic and specific use the avro.java.string setting
    String name = schema.getProp(STRINGABLE_PROP);
    if (name == null) {
      return CharSequence.class;
    }

    switch (GenericData.StringType.valueOf(name)) {
      case String:
        return String.class;
      default:
        return CharSequence.class; // will use Utf8
    }
  }

  private static <T> Class<T> getDatumClass(Schema schema, GenericData model) {
    return getDatumClass(null, null, schema, model);
  }

  @SuppressWarnings("unchecked")
  private static <T> Class<T> getDatumClass(Conversion<?> conversion,
                                            Class<T> knownClass,
                                            Schema schema, GenericData model) {
    if (conversion != null) {
      // use generic classes to pass data to conversions
      return null;
    }

    // known class can be set when using reflect
    if (knownClass != null) {
      return knownClass;
    }

    if (model instanceof SpecificData) {
      // this works for reflect as well
      return ((SpecificData) model).getClass(schema);

    } else if (model.getClass() == GenericData.class) {
      return null;

    } else {
      // try to use reflection (for ThriftData and others)
      Class<? extends GenericData> modelClass = model.getClass();
      Method getClassMethod;
      try {
        getClassMethod = modelClass.getMethod("getClass", Schema.class);
      } catch (NoSuchMethodException e) {
        return null; // no getClass method
      }

      try {
        return (Class<T>) getClassMethod.invoke(schema);
      } catch (IllegalAccessException e) {
        return null;
      } catch (InvocationTargetException e) {
        return null;
      }
    }
  }

  protected void set(String name, int avroIndex, Object value) {
    model.setField(currentRecord, name, avroIndex, value);
  }

  @Override
  public Converter getConverter(int fieldIndex) {
    return converters[fieldIndex];
  }

  @Override
  @SuppressWarnings("unchecked")
  public void start() {
    this.currentRecord = (T) model.newRecord(null, avroSchema);
  }

  @Override
  public void end() {
    fillInDefaults();
    if (parent != null) {
      parent.add(currentRecord);
    } else {
      // this applies any converters needed for the root value
      rootContainer.add(currentRecord);
    }
  }

  private void fillInDefaults() {
    for (Map.Entry<Schema.Field, Object> entry : recordDefaults.entrySet()) {
      Schema.Field f = entry.getKey();
      // replace following with model.deepCopy once AVRO-1455 is being used
      Object defaultValue = deepCopy(f.schema(), entry.getValue());
      set(f.name(), f.pos(), defaultValue);
    }
  }

  private Object deepCopy(Schema schema, Object value) {
    switch (schema.getType()) {
      case BOOLEAN:
      case INT:
      case LONG:
      case FLOAT:
      case DOUBLE:
        return value;
      default:
        return model.deepCopy(schema, value);
    }
  }

  T getCurrentRecord() {
    return currentRecord;
  }

  /**
   * Converter for a list to a Java Collection.
   *
   *

   *   optional group the_list (LIST) { <-- this layer
   *     repeated group array {
   *       optional (type) element;
   *     }
   *   }
   *

   *
   * This class also implements LIST element backward-compatibility rules.
   */
  static final class AvroCollectionConverter extends GroupConverter {

    private final ParentValueContainer parent;
    private final Schema avroSchema;
    private final Converter converter;
    private Class<?> containerClass;
    private Collection<Object> container;

    public AvroCollectionConverter(ParentValueContainer parent, GroupType type,
                                   Schema avroSchema, GenericData model,
                                   Class<?> containerClass) {
      this.parent = parent;
      this.avroSchema = avroSchema;
      this.containerClass = containerClass;
      Schema elementSchema = AvroSchemaConverter.getNonNull(avroSchema.getElementType());
      Type repeatedType = type.getType(0);
      // always determine whether the repeated type is the element type by
      // matching it against the element schema.
      if (isElementType(repeatedType, elementSchema)) {
        // the element type is the repeated type (and required)
        converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() {
          @Override
          @SuppressWarnings("unchecked")
          public void add(Object value) {
            container.add(value);
          }
        });
      } else {
        // the element is wrapped in a synthetic group and may be optional
        converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model);
      }
    }

    @Override
    public Converter getConverter(int fieldIndex) {
      return converter;
    }

    @Override
    public void start() {
      container = newContainer();
    }

    @Override
    public void end() {
      parent.add(container);
    }

    @SuppressWarnings("unchecked")
    private Collection<Object> newContainer() {
      if (containerClass == null) {
        return new GenericData.Array<Object>(0, avroSchema);
      } else if (containerClass.isAssignableFrom(ArrayList.class)) {
        return new ArrayList<Object>();
      } else {
        // not need to use the data model to instantiate because it resolved
        // the class, which used the correct ClassLoader
        return (Collection<Object>) ReflectData.newInstance(containerClass, avroSchema);
      }
    }

    /**
     * Converter for list elements.
     *
     *

     *   optional group the_list (LIST) {
     *     repeated group array { <-- this layer
     *       optional (type) element;
     *     }
     *   }
     *

     */
    final class ElementConverter extends GroupConverter {
      private Object element;
      private final Converter elementConverter;

      public ElementConverter(GroupType repeatedType, Schema elementSchema, GenericData model) {
        Type elementType = repeatedType.getType(0);
        Schema nonNullElementSchema = AvroSchemaConverter.getNonNull(elementSchema);
        this.elementConverter = newConverter(nonNullElementSchema, elementType, model, new ParentValueContainer() {
          @Override
          @SuppressWarnings("unchecked")
          public void add(Object value) {
            ElementConverter.this.element = value;
          }
        });
      }

      @Override
      public Converter getConverter(int fieldIndex) {
        Preconditions.checkArgument(
            fieldIndex == 0, "Illegal field index: " + fieldIndex);
        return elementConverter;
      }

      @Override
      public void start() {
        element = null;
      }

      @Override
      public void end() {
        container.add(element);
      }
    }
  }

  /**
   * Converter for a list to a Java array.
   *
   *

   *   optional group the_list (LIST) { <-- this layer
   *     repeated group array {
   *       optional (type) element;
   *     }
   *   }
   *

   *
   * This class also implements LIST element backward-compatibility rules.
   */
  static final class AvroArrayConverter extends GroupConverter {

    private final ParentValueContainer parent;
    private final Schema avroSchema;
    private final Converter converter;
    private Class<?> elementClass;
    private Collection<?> container;

    public AvroArrayConverter(ParentValueContainer parent, GroupType type,
                              Schema avroSchema, GenericData model,
                              Class<?> arrayClass) {
      this.parent = parent;
      this.avroSchema = avroSchema;

      Preconditions.checkArgument(arrayClass.isArray(),
          "Cannot convert non-array: " + arrayClass.getName());
      this.elementClass = arrayClass.getComponentType();

      ParentValueContainer setter = createSetterAndContainer();
      Schema elementSchema = this.avroSchema.getElementType();
      Type repeatedType = type.getType(0);

      // always determine whether the repeated type is the element type by
      // matching it against the element schema.
      if (isElementType(repeatedType, elementSchema)) {
        // the element type is the repeated type (and required)
        converter = newConverter(elementSchema, repeatedType, model, elementClass, setter);
      } else {
        // the element is wrapped in a synthetic group and may be optional
        converter = new ArrayElementConverter(
            repeatedType.asGroupType(), elementSchema, model, setter);
      }
    }

    @Override
    public Converter getConverter(int fieldIndex) {
      return converter;
    }

    @Override
    public void start() {
      // end creates a new copy of the array so the container is safe to reuse
      container.clear();
    }

    @Override
    public void end() {
      if (elementClass == boolean.class) {
        parent.add(((BooleanArrayList) container).toBooleanArray());
      } else if (elementClass == byte.class) {
        parent.add(((ByteArrayList) container).toByteArray());
      } else if (elementClass == char.class) {
        parent.add(((CharArrayList) container).toCharArray());
      } else if (elementClass == short.class) {
        parent.add(((ShortArrayList) container).toShortArray());
      } else if (elementClass == int.class) {
        parent.add(((IntArrayList) container).toIntArray());
      } else if (elementClass == long.class) {
        parent.add(((LongArrayList) container).toLongArray());
      } else if (elementClass == float.class) {
        parent.add(((FloatArrayList) container).toFloatArray());
      } else if (elementClass == double.class) {
        parent.add(((DoubleArrayList) container).toDoubleArray());
      } else {
        parent.add(((ArrayList) container).toArray());
      }
    }

    @SuppressWarnings("unchecked")
    private ParentValueContainer createSetterAndContainer() {
      if (elementClass == boolean.class) {
        final BooleanArrayList list = new BooleanArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addBoolean(boolean value) {
            list.add(value);
          }
        };
      } else if (elementClass == byte.class) {
        final ByteArrayList list = new ByteArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addByte(byte value) {
            list.add(value);
          }
        };
      } else if (elementClass == char.class) {
        final CharArrayList list = new CharArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addChar(char value) {
            list.add(value);
          }
        };
      } else if (elementClass == short.class) {
        final ShortArrayList list = new ShortArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addShort(short value) {
            list.add(value);
          }
        };
      } else if (elementClass == int.class) {
        final IntArrayList list = new IntArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addInt(int value) {
            list.add(value);
          }
        };
      } else if (elementClass == long.class) {
        final LongArrayList list = new LongArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addLong(long value) {
            list.add(value);
          }
        };
      } else if (elementClass == float.class) {
        final FloatArrayList list = new FloatArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addFloat(float value) {
            list.add(value);
          }
        };
      } else if (elementClass == double.class) {
        final DoubleArrayList list = new DoubleArrayList();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void addDouble(double value) {
            list.add(value);
          }
        };
      } else {
        // this will end up as Object[]
        final List<Object> list = new ArrayList<Object>();
        this.container = list;
        return new ParentValueContainer() {
          @Override
          public void add(Object value) {
            list.add(value);
          }
        };
      }

    }

    /**
     * Converter for primitive list elements.
     *
     *

     *   optional group the_list (LIST) {
     *     repeated group array { <-- this layer
     *       optional (type) element;
     *     }
     *   }
     *

     */
    final class ArrayElementConverter extends GroupConverter {
      private boolean isSet;
      private final Converter elementConverter;

      public ArrayElementConverter(GroupType repeatedType,
                                   Schema elementSchema, GenericData model,
                                   final ParentValueContainer setter) {
        Type elementType = repeatedType.getType(0);
        Preconditions.checkArgument(
            !elementClass.isPrimitive() || elementType.isRepetition(REQUIRED),
            "Cannot convert list of optional elements to primitive array");
        Schema nonNullElementSchema = AvroSchemaConverter.getNonNull(elementSchema);
        this.elementConverter = newConverter(
            nonNullElementSchema, elementType, model, elementClass, new ParentValueContainer() {
              @Override
              public void add(Object value) {
                isSet = true;
                setter.add(value);
              }

              @Override
              public void addByte(byte value) {
                isSet = true;
                setter.addByte(value);
              }

              @Override
              public void addBoolean(boolean value) {
                isSet = true;
                setter.addBoolean(value);
              }

              @Override
              public void addChar(char value) {
                isSet = true;
                setter.addChar(value);
              }

              @Override
              public void addShort(short value) {
                isSet = true;
                setter.addShort(value);
              }

              @Override
              public void addInt(int value) {
                isSet = true;
                setter.addInt(value);
              }

              @Override
              public void addLong(long value) {
                isSet = true;
                setter.addLong(value);
              }

              @Override
              public void addFloat(float value) {
                isSet = true;
                setter.addFloat(value);
              }

              @Override
              public void addDouble(double value) {
                isSet = true;
                setter.addDouble(value);
              }
            });
      }

      @Override
      public Converter getConverter(int fieldIndex) {
        Preconditions.checkArgument(
            fieldIndex == 0, "Illegal field index: " + fieldIndex);
        return elementConverter;
      }

      @Override
      public void start() {
        isSet = false;
      }

      @Override
      public void end() {
        if (!isSet) {
          container.add(null);
        }
      }
    }
  }

  // Converter used to test whether a requested schema is a 2-level schema.
  // This is used to convert the file's type assuming that the file uses
  // 2-level lists and the result is checked to see if it matches the requested
  // element type. This should always convert assuming 2-level lists because
  // 2-level and 3-level can't be mixed.
  private static final AvroSchemaConverter CONVERTER =
      new AvroSchemaConverter(true);

  /**
   * Returns whether the given type is the element type of a list or is a
   * synthetic group with one field that is the element type. This is
   * determined by checking whether the type can be a synthetic group and by
   * checking whether a potential synthetic group matches the expected schema.
   *

   * Unlike {@link AvroSchemaConverter#isElementType(Type, String)}, this
   * method never guesses because the expected schema is known.
   *
   * @param repeatedType a type that may be the element type
   * @param elementSchema the expected Schema for list elements
   * @return {@code true} if the repeatedType is the element schema
   */
  static boolean isElementType(Type repeatedType, Schema elementSchema) {
    if (repeatedType.isPrimitive() ||
        repeatedType.asGroupType().getFieldCount() > 1 ||
        repeatedType.asGroupType().getType(0).isRepetition(REPEATED)) {
      // The repeated type must be the element type because it is an invalid
      // synthetic wrapper. Must be a group with one optional or required field
      return true;
    } else if (elementSchema != null &&
        elementSchema.getType() == Schema.Type.RECORD) {
      Schema schemaFromRepeated = CONVERTER.convert(repeatedType.asGroupType());
      if (checkReaderWriterCompatibility(elementSchema, schemaFromRepeated)
          .getType() == COMPATIBLE) {
        return true;
      }
    }
    return false;
  }

  static final class AvroUnionConverter extends AvroConverters.AvroGroupConverter {
    private final Converter[] memberConverters;
    private Object memberValue = null;

    public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                              Schema avroSchema, GenericData model) {
      super(parent);
      GroupType parquetGroup = parquetSchema.asGroupType();
      this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

      int parquetIndex = 0;
      for (int index = 0; index < avroSchema.getTypes().size(); index++) {
        Schema memberSchema = avroSchema.getTypes().get(index);
        if (!memberSchema.getType().equals(Schema.Type.NULL)) {
          Type memberType = parquetGroup.getType(parquetIndex);
          memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
            @Override
            public void add(Object value) {
              Preconditions.checkArgument(
                  AvroUnionConverter.this.memberValue == null,
                  "Union is resolving to more than one type");
              memberValue = value;
            }
          });
          parquetIndex++; // Note for nulls the parquetIndex id not increased
        }
      }
    }

    @Override
    public Converter getConverter(int fieldIndex) {
      return memberConverters[fieldIndex];
    }

    @Override
    public void start() {
      memberValue = null;
    }

    @Override
    public void end() {
      parent.add(memberValue);
    }
  }

  static final class MapConverter<K, V> extends GroupConverter {

    private final ParentValueContainer parent;
    private final Converter keyValueConverter;
    private final Schema schema;
    private final Class<?> mapClass;
    private Map<K, V> map;

    public MapConverter(ParentValueContainer parent, GroupType mapType,
        Schema mapSchema, GenericData model) {
      this.parent = parent;
      GroupType repeatedKeyValueType = mapType.getType(0).asGroupType();
      this.keyValueConverter = new MapKeyValueConverter(
          repeatedKeyValueType, mapSchema, model);
      this.schema = mapSchema;
      this.mapClass = getDatumClass(mapSchema, model);
    }

    @Override
    public Converter getConverter(int fieldIndex) {
      return keyValueConverter;
    }

    @Override
    public void start() {
      this.map = newMap();
    }

    @Override
    public void end() {
      parent.add(map);
    }

    @SuppressWarnings("unchecked")
    private Map<K, V> newMap() {
      if (mapClass == null || mapClass.isAssignableFrom(HashMap.class)) {
        return new HashMap<K, V>();
      } else {
        return (Map<K, V>) ReflectData.newInstance(mapClass, schema);
      }
    }

    final class MapKeyValueConverter extends GroupConverter {

      private K key;
      private V value;
      private final Converter keyConverter;
      private final Converter valueConverter;

      public MapKeyValueConverter(GroupType keyValueType, Schema mapSchema,
          GenericData model) {
        keyConverter = newStringConverter(mapSchema, model,
            new ParentValueContainer() {
              @Override
              @SuppressWarnings("unchecked")
              public void add(Object value) {
                MapKeyValueConverter.this.key = (K) value;
              }
            });

        Type valueType = keyValueType.getType(1);
        Schema nonNullValueSchema = AvroSchemaConverter.getNonNull(mapSchema.getValueType());
        valueConverter = newConverter(nonNullValueSchema, valueType, model, new ParentValueContainer() {
          @Override
          @SuppressWarnings("unchecked")
          public void add(Object value) {
            MapKeyValueConverter.this.value = (V) value;
          }
        });
      }

      @Override
      public Converter getConverter(int fieldIndex) {
        if (fieldIndex == 0) {
          return keyConverter;
        } else if (fieldIndex == 1) {
          return valueConverter;
        }
        throw new IllegalArgumentException("only the key (0) and value (1) fields expected: " + fieldIndex);
      }

      @Override
      public void start() {
        key = null;
        value = null;
      }

      @Override
      public void end() {
        map.put(key, value);
      }
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.thrift;

import java.nio.ByteBuffer;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TMessage;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TStruct;
import org.apache.thrift.protocol.TType;

import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
import org.apache.parquet.io.ParquetEncodingException;
import org.apache.parquet.io.PrimitiveColumnIO;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.thrift.struct.ThriftField;
import org.apache.parquet.thrift.struct.ThriftType;
import org.apache.parquet.thrift.struct.ThriftType.EnumType;
import org.apache.parquet.thrift.struct.ThriftType.EnumValue;
import org.apache.parquet.thrift.struct.ThriftType.ListType;
import org.apache.parquet.thrift.struct.ThriftType.MapType;
import org.apache.parquet.thrift.struct.ThriftType.SetType;
import org.apache.parquet.thrift.struct.ThriftType.StructType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ParquetWriteProtocol extends ParquetProtocol {

  interface Events {

    public void start();

    public void end();

  }

  abstract class FieldBaseWriteProtocol extends ParquetProtocol {
    private final Events returnClause;

    public FieldBaseWriteProtocol(Events returnClause) {
      this.returnClause = returnClause;
    }

    void start() {
      this.returnClause.start();
    }

    void end() {
      this.returnClause.end();
    }
  }

  class EnumWriteProtocol extends FieldBaseWriteProtocol {

    private final EnumType type;
    private PrimitiveColumnIO columnIO;
    public EnumWriteProtocol(PrimitiveColumnIO columnIO, EnumType type, Events returnClause) {
      super(returnClause);
      this.columnIO = columnIO;
      this.type = type;
    }

    @Override
    public void writeI32(int i32) throws TException {
      start();
      EnumValue value = type.getEnumValueById(i32);
      if (value == null) {
        throw new ParquetEncodingException("Can not find enum value of index " + i32 + " for field:" + columnIO.toString());
      }
      recordConsumer.addBinary(Binary.fromString(value.getName()));
      end();
    }

  }

  class ListWriteProtocol extends FieldBaseWriteProtocol {

    private ColumnIO listContent;
    private TProtocol contentProtocol;
    private int size;

    public ListWriteProtocol(GroupColumnIO columnIO, ThriftField values, Events returnClause) {
      super(returnClause);
      this.listContent = columnIO.getChild(0);
      this.contentProtocol = getProtocol(values, listContent, new Events() {
        int consumedRecords = 0;
        @Override
        public void start() {
        }

        @Override
        public void end() {
          ++ consumedRecords;
          if (consumedRecords == size) {
            currentProtocol = ListWriteProtocol.this;
            consumedRecords = 0;
          }
        }
      });
    }

    private void startListWrapper() {
      start();
      recordConsumer.startGroup();
      if (size > 0) {
        recordConsumer.startField(listContent.getType().getName(), 0);
        currentProtocol = contentProtocol;
      }
    }

    private void endListWrapper() {
      if (size > 0) {
        recordConsumer.endField(listContent.getType().getName(), 0);
      }
      recordConsumer.endGroup();
      end();
    }

    @Override
    public void writeListBegin(TList list) throws TException {
      size = list.size;
      startListWrapper();
    }

    @Override
    public void writeListEnd() throws TException {
      endListWrapper();
    }

    @Override
    public void writeSetBegin(TSet set) throws TException {
      size = set.size;
      startListWrapper();
    }

    @Override
    public void writeSetEnd() throws TException {
      endListWrapper();
    }

  }

  class MapWriteProtocol extends FieldBaseWriteProtocol {

    private GroupColumnIO mapContent;
    private ColumnIO key;
    private ColumnIO value;
    private TProtocol keyProtocol;
    private TProtocol valueProtocol;
    private int countToConsume;

    public MapWriteProtocol(GroupColumnIO columnIO, MapType type, Events returnClause) {
      super(returnClause);
      this.mapContent = (GroupColumnIO)columnIO.getChild(0);
      this.key = mapContent.getChild(0);
      this.value = mapContent.getChild(1);
      this.keyProtocol = getProtocol(type.getKey(), this.key, new Events() {
        @Override
        public void start() {
          recordConsumer.startGroup();
          recordConsumer.startField(key.getName(), key.getIndex());
        }
        @Override
        public void end() {
          recordConsumer.endField(key.getName(), key.getIndex());
          currentProtocol = valueProtocol;
        }
      });
      this.valueProtocol = getProtocol(type.getValue(), this.value, new Events() {
        int consumed;
        @Override
        public void start() {
          recordConsumer.startField(value.getName(), value.getIndex());
        }
        @Override
        public void end() {
          consumed ++;
          recordConsumer.endField(value.getName(), value.getIndex());
          recordConsumer.endGroup();
          if (consumed == countToConsume) {
            currentProtocol = MapWriteProtocol.this;
            consumed = 0;
          } else {
            currentProtocol = keyProtocol;
          }

        }
      });
    }

    @Override
    public void writeMapBegin(TMap map) throws TException {
      start();
      recordConsumer.startGroup();
      countToConsume = map.size;
      if (countToConsume > 0) {
        recordConsumer.startField(mapContent.getType().getName(), 0);
        currentProtocol = keyProtocol;
      }
    }

    @Override
    public void writeMapEnd() throws TException {
      if (countToConsume > 0) {
        recordConsumer.endField(mapContent.getType().getName(), 0);
      }
      recordConsumer.endGroup();
      end();
    }

  }

  class PrimitiveWriteProtocol extends FieldBaseWriteProtocol {

    public PrimitiveWriteProtocol(PrimitiveColumnIO columnIO, Events returnClause) {
      super(returnClause);
    }

    @Override
    public void writeBool(boolean b) throws TException {
      start();
      recordConsumer.addBoolean(b);
      end();
    }

    @Override
    public void writeByte(byte b) throws TException {
      start();
      recordConsumer.addInteger(b);
      end();
    }

    @Override
    public void writeI16(short i16) throws TException {
      start();
      recordConsumer.addInteger(i16);
      end();
    }

    @Override
    public void writeI32(int i32) throws TException {
      start();
      recordConsumer.addInteger(i32);
      end();
    }

    @Override
    public void writeI64(long i64) throws TException {
      start();
      recordConsumer.addLong(i64);
      end();
    }

    @Override
    public void writeDouble(double dub) throws TException {
      start();
      recordConsumer.addDouble(dub);
      end();
    }

    @Override
    public void writeString(String str) throws TException {
      start();
      writeStringToRecordConsumer(str);
      end();
    }

    @Override
    public void writeBinary(ByteBuffer buf) throws TException {
      start();
      writeBinaryToRecordConsumer(buf);
      end();
    }

  }

  class StructWriteProtocol extends FieldBaseWriteProtocol {

    private final GroupColumnIO schema;
    private final StructType thriftType;
    private final TProtocol[] children;
    private ColumnIO currentType;
    private ColumnIO[] thriftFieldIdToParquetField;

    public StructWriteProtocol(GroupColumnIO schema, StructType thriftType, Events returnClause) {
      super(returnClause);
      if (schema == null) {
        throw new NullPointerException("schema");
      }
      this.thriftType = thriftType;
      int maxFieldId = 0;
      for (ThriftField field : thriftType.getChildren()) {
        maxFieldId = Math.max(maxFieldId, field.getFieldId());
      }
      thriftFieldIdToParquetField = new ColumnIO[maxFieldId + 1];
      for (int i = 0; i < thriftType.getChildren().size(); i++) {
        thriftFieldIdToParquetField[thriftType.getChildren().get(i).getFieldId()] = schema.getChild(i);
      }
      for (ThriftField field : thriftType.getChildren()) {
      }
      this.schema = schema;
      children = new TProtocol[thriftType.getChildren().size()];
      for (int i = 0; i < children.length; i++) {
        final ThriftField field = thriftType.getChildren().get(i);
        final ColumnIO columnIO = schema.getChild(field.getName());
        if (columnIO == null) {
          throw new RuntimeException("Could not find " + field.getName() + " in " + schema);
        }
        try {
          TProtocol p;
          p = getProtocol(field, columnIO, new Events() {
            @Override
            public void start() {
            }

            @Override
            public void end() {
              currentProtocol = StructWriteProtocol.this;
            }
          });
          children[i] = p;
        } catch (RuntimeException e) {
          throw new ParquetEncodingException("Could not create Protocol for " + field + " to " + columnIO, e);
        }
      }
    }

    @Override
    public void writeStructBegin(TStruct struct) throws TException {
      start();
      recordConsumer.startGroup();
    }

    @Override
    public void writeStructEnd() throws TException {
      recordConsumer.endGroup();
      end();
    }

    @Override
    public void writeFieldBegin(TField field) throws TException {
      if (field.type == TType.STOP) {
        return;
      }
      try {
        currentType = thriftFieldIdToParquetField[field.id];
        if (currentType == null) {
          throw new ParquetEncodingException("field " + field.id + " was not found in " + thriftType + " and " + schema.getType());
        }
        final int index = currentType.getIndex();
        recordConsumer.startField(currentType.getName(), index);
        currentProtocol = children[index];
      } catch (ArrayIndexOutOfBoundsException e) {
        throw new ParquetEncodingException("field " + field.id + " was not found in " + thriftType + " and " + schema.getType());
      }
    }

    @Override
    public void writeFieldStop() throws TException {
      // duplicate with struct end
    }

    @Override
    public void writeFieldEnd() throws TException {
      recordConsumer.endField(currentType.getName(), currentType.getIndex());
    }
  }

  class MessageWriteProtocol extends StructWriteProtocol {

    public MessageWriteProtocol(MessageColumnIO schema, StructType thriftType) {
      super(schema, thriftType, null);
    }

    @Override
    public void writeStructBegin(TStruct struct) throws TException {
      recordConsumer.startMessage();
    }

    @Override
    public void writeStructEnd() throws TException {
      recordConsumer.endMessage();
    }

  }

  private static final Logger LOG = LoggerFactory.getLogger(ParquetWriteProtocol.class);

  private final RecordConsumer recordConsumer;
  private TProtocol currentProtocol;

  private String toString(TStruct struct) {
    return "<TStruct name:" + struct.name + ">";
  }

  private String toString(TList list) {
    return "<TList elemType:" + list.elemType + " size:" + list.size + ">";
  }

  private String toString(TMap map) {
    return "<TMap keyType:" + map.keyType + " valueType:" + map.valueType + " size:" + map.size + ">";
  }

  public ParquetWriteProtocol(RecordConsumer recordConsumer, MessageColumnIO schema, StructType thriftType) {
    this.currentProtocol = new MessageWriteProtocol(schema, thriftType);
    this.recordConsumer = recordConsumer;
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeMessageBegin(org.apache.thrift.protocol.TMessage)
   */
  @Override
  public void writeMessageBegin(TMessage message) throws TException {
    LOG.debug("writeMessageBegin({})", message);
    currentProtocol.writeMessageBegin(message);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeMessageEnd()
   */
  @Override
  public void writeMessageEnd() throws TException {
    LOG.debug("writeMessageEnd()");
    currentProtocol.writeMessageEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeStructBegin(org.apache.thrift.protocol.TStruct)
   */
  @Override
  public void writeStructBegin(TStruct struct) throws TException {
    if (LOG.isDebugEnabled()) LOG.debug("writeStructBegin("+toString(struct)+")");
    currentProtocol.writeStructBegin(struct);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeStructEnd()
   */
  @Override
  public void writeStructEnd() throws TException {
    LOG.debug("writeStructEnd()");
    currentProtocol.writeStructEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeFieldBegin(org.apache.thrift.protocol.TField)
   */
  @Override
  public void writeFieldBegin(TField field) throws TException {
    LOG.debug("writeFieldBegin({})", field);
    currentProtocol.writeFieldBegin(field);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeFieldEnd()
   */
  @Override
  public void writeFieldEnd() throws TException {
    LOG.debug("writeFieldEnd()");
    currentProtocol.writeFieldEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeFieldStop()
   */
  @Override
  public void writeFieldStop() throws TException {
    LOG.debug("writeFieldStop()");
    currentProtocol.writeFieldStop();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeMapBegin(org.apache.thrift.protocol.TMap)
   */
  @Override
  public void writeMapBegin(TMap map) throws TException {
    if (LOG.isDebugEnabled()) LOG.debug("writeMapBegin("+toString(map)+")");
    currentProtocol.writeMapBegin(map);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeMapEnd()
   */
  @Override
  public void writeMapEnd() throws TException {
    LOG.debug("writeMapEnd()");
    currentProtocol.writeMapEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeListBegin(org.apache.thrift.protocol.TList)
   */
  @Override
  public void writeListBegin(TList list) throws TException {
    if (LOG.isDebugEnabled()) LOG.debug("writeListBegin("+toString(list)+")");
    currentProtocol.writeListBegin(list);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeListEnd()
   */
  @Override
  public void writeListEnd() throws TException {
    LOG.debug("writeListEnd()");
    currentProtocol.writeListEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeSetBegin(org.apache.thrift.protocol.TSet)
   */
  @Override
  public void writeSetBegin(TSet set) throws TException {
    LOG.debug("writeSetBegin({})", set);
    currentProtocol.writeSetBegin(set);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeSetEnd()
   */
  @Override
  public void writeSetEnd() throws TException {
    LOG.debug("writeSetEnd()");
    currentProtocol.writeSetEnd();
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeBool(boolean)
   */
  @Override
  public void writeBool(boolean b) throws TException {
    LOG.debug("writeBool({})", b);
    currentProtocol.writeBool(b);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeByte(byte)
   */
  @Override
  public void writeByte(byte b) throws TException {
    LOG.debug("writeByte({})", b);
    currentProtocol.writeByte(b);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeI16(short)
   */
  @Override
  public void writeI16(short i16) throws TException {
    LOG.debug("writeI16({})", i16);
    currentProtocol.writeI16(i16);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeI32(int)
   */
  @Override
  public void writeI32(int i32) throws TException {
    LOG.debug("writeI32({})", i32);
    currentProtocol.writeI32(i32);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeI64(long)
   */
  @Override
  public void writeI64(long i64) throws TException {
    LOG.debug("writeI64({})", i64);
    currentProtocol.writeI64(i64);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeDouble(double)
   */
  @Override
  public void writeDouble(double dub) throws TException {
    LOG.debug("writeDouble({})", dub);
    currentProtocol.writeDouble(dub);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeString(java.lang.String)
   */
  @Override
  public void writeString(String str) throws TException {
    LOG.debug("writeString({})", str);
    currentProtocol.writeString(str);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.thrift.ParquetProtocol#writeBinary(java.nio.ByteBuffer)
   */
  @Override
  public void writeBinary(ByteBuffer buf) throws TException {
    LOG.debug("writeBinary({})", buf);
    currentProtocol.writeBinary(buf);
  }

  private void writeBinaryToRecordConsumer(ByteBuffer buf) {
    recordConsumer.addBinary(Binary.fromReusedByteArray(buf.array(), buf.position(),
        buf.limit() - buf.position()));
  }

  private void writeStringToRecordConsumer(String str) {
    recordConsumer.addBinary(Binary.fromString(str));
  }

  private TProtocol getProtocol(ThriftField field, ColumnIO columnIO, Events returnClause) {
    TProtocol p;
    final ThriftType type = field.getType();
    switch (type.getType()) {
    case STOP:
    case VOID:
    default:
      throw new UnsupportedOperationException("can't convert type of " + field);
    case BOOL:
    case BYTE:
    case DOUBLE:
    case I16:
    case I32:
    case I64:
    case STRING:
      p = new PrimitiveWriteProtocol((PrimitiveColumnIO)columnIO, returnClause);
      break;
    case STRUCT:
      p = new StructWriteProtocol((GroupColumnIO)columnIO, (StructType)type, returnClause);
      break;
    case MAP:
      p = new MapWriteProtocol((GroupColumnIO)columnIO, (MapType)type, returnClause);
      break;
    case SET:
      p = new ListWriteProtocol((GroupColumnIO)columnIO, ((SetType)type).getValues(), returnClause);
      break;
    case LIST:
      p = new ListWriteProtocol((GroupColumnIO)columnIO, ((ListType)type).getValues(), returnClause);
      break;
    case ENUM:
      p = new EnumWriteProtocol((PrimitiveColumnIO)columnIO, (EnumType)type, returnClause);
      break;
    }
    return p;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hadoop.hive.ql.io.parquet.read;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.parquet.convert.DataWritableRecordConverter;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.io.ArrayWritable;

import org.apache.parquet.hadoop.api.ReadSupport;
import org.apache.parquet.hive.HiveBindingFactory;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;

/**
 * A MapWritableReadSupport
 *
 * Manages the translation between Hive and Parquet
 */
public class DataWritableReadSupport extends ReadSupport<ArrayWritable> {

  private static final String TABLE_SCHEMA = "table_schema";
  public static final String HIVE_SCHEMA_KEY = "HIVE_TABLE_SCHEMA";
  public static final String PARQUET_COLUMN_INDEX_ACCESS = "parquet.column.index.access";

  /**
   * From a string which columns names (including hive column), return a list
   * of string columns
   *
   * @param columns comma separated list of columns
   * @return list with virtual columns removed
   */
  private static List<String> getColumns(final String columns) {
    return (new HiveBindingFactory()).create().getColumns(columns);
  }
  /**
   *
   * It creates the readContext for Parquet side with the requested schema during the init phase.
   *
   * @param configuration needed to get the wanted columns
   * @param keyValueMetaData // unused
   * @param fileSchema parquet file schema
   * @return the parquet ReadContext
   */
  @Override
  public org.apache.parquet.hadoop.api.ReadSupport.ReadContext init(final Configuration configuration,
      final Map<String, String> keyValueMetaData, final MessageType fileSchema) {
    final String columns = configuration.get(IOConstants.COLUMNS);
    final Map<String, String> contextMetadata = new HashMap<String, String>();
    if (columns != null) {
      final List<String> listColumns = getColumns(columns);

      final List<Type> typeListTable = new ArrayList<Type>();
      for (final String col : listColumns) {
        // listColumns contains partition columns which are metadata only
        if (fileSchema.containsField(col)) {
          typeListTable.add(fileSchema.getType(col));
        } else {
          // below allows schema evolution
          typeListTable.add(new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, col));
        }
      }
      MessageType tableSchema = new MessageType(TABLE_SCHEMA, typeListTable);
      contextMetadata.put(HIVE_SCHEMA_KEY, tableSchema.toString());

      MessageType requestedSchemaByUser = tableSchema;
      final List<Integer> indexColumnsWanted = ColumnProjectionUtils.getReadColumnIDs(configuration);

      final List<Type> typeListWanted = new ArrayList<Type>();
      for (final Integer idx : indexColumnsWanted) {
        typeListWanted.add(tableSchema.getType(listColumns.get(idx)));
      }
      requestedSchemaByUser = resolveSchemaAccess(new MessageType(fileSchema.getName(),
              typeListWanted), fileSchema, configuration);

      return new ReadContext(requestedSchemaByUser, contextMetadata);
    } else {
      contextMetadata.put(HIVE_SCHEMA_KEY, fileSchema.toString());
      return new ReadContext(fileSchema, contextMetadata);
    }
  }

  /**
   *
   * It creates the hive read support to interpret data from parquet to hive
   *
   * @param configuration // unused
   * @param keyValueMetaData string map of metadata
   * @param fileSchema // unused
   * @param readContext containing the requested schema and the schema of the hive table
   * @return Record Materialize for Hive
   */
  @Override
  public RecordMaterializer<ArrayWritable> prepareForRead(final Configuration configuration,
      final Map<String, String> keyValueMetaData, final MessageType fileSchema,
          final org.apache.parquet.hadoop.api.ReadSupport.ReadContext readContext) {
    final Map<String, String> metadata = readContext.getReadSupportMetadata();
    if (metadata == null) {
      throw new IllegalStateException("ReadContext not initialized properly. " +
        "Don't know the Hive Schema.");
    }
    final MessageType tableSchema = resolveSchemaAccess(MessageTypeParser.
        parseMessageType(metadata.get(HIVE_SCHEMA_KEY)), fileSchema, configuration);

    return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema);
  }

  /**
  * Determine the file column names based on the position within the requested columns and
  * use that as the requested schema.
  */
  private MessageType resolveSchemaAccess(MessageType requestedSchema, MessageType fileSchema,
          Configuration configuration) {
    if(configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false)) {
      final List<String> listColumns = getColumns(configuration.get(IOConstants.COLUMNS));

      List<Type> requestedTypes = new ArrayList<Type>();

      for(Type t : requestedSchema.getFields()) {
        int index = listColumns.indexOf(t.getName());
        requestedTypes.add(fileSchema.getType(index));
      }

      requestedSchema = new MessageType(requestedSchema.getName(), requestedTypes);
    }

    return requestedSchema;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.io;

import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.RecordConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;

/**
 * This class can be used to wrap an actual RecordConsumer and log all calls
 */
public class RecordConsumerLoggingWrapper extends RecordConsumer {
    private static final Logger LOG = LoggerFactory.getLogger(RecordConsumerLoggingWrapper.class);

    private final RecordConsumer delegate;

    int indent = 0;

    /**
     * all calls a delegate to the wrapped delegate
     * @param delegate a wrapped record consumer that does the real work
     */
    public RecordConsumerLoggingWrapper(RecordConsumer delegate) {
      this.delegate = delegate;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void startField(String field, int index) {
      logOpen(field);
      delegate.startField(field, index);
    }

    private void logOpen(String field) {
      log("<{}>", field);
    }

    private String indent() {
      StringBuilder result = new StringBuilder();
      for (int i = 0; i < indent; i++) {
        result.append("  ");
      }
      return result.toString();
    }

    private void log(Object value, Object ... parameters) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(indent() + value, parameters);
      }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void startGroup() {
      ++indent;
      log("<!-- start group -->");
      delegate.startGroup();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addInteger(int value) {
      log(value);
      delegate.addInteger(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addLong(long value) {
      log(value);
      delegate.addLong(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addBoolean(boolean value) {
      log(value);
      delegate.addBoolean(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addBinary(Binary value) {
      if (LOG.isDebugEnabled()) log(Arrays.toString(value.getBytesUnsafe()));
      delegate.addBinary(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addFloat(float value) {
      log(value);
      delegate.addFloat(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addDouble(double value) {
      log(value);
      delegate.addDouble(value);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void flush() {
      log("<!-- flush -->");
      delegate.flush();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void endGroup() {
      log("<!-- end group -->");
      --indent;
      delegate.endGroup();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void endField(String field, int index) {
      logClose(field);
      delegate.endField(field, index);
    }

    private void logClose(String field) {
      log("</{}>", field);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void startMessage() {
      log("<!-- start message -->");
      delegate.startMessage();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void endMessage() {
      delegate.endMessage();
      log("<!-- end message -->");
    }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.tools.command;

import org.apache.commons.cli.CommandLine;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.parquet.hadoop.util.HiddenFileFilter;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.metadata.FileMetaData;
import org.apache.parquet.tools.Main;

import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class MergeCommand extends ArgsOnlyCommand {
  public static final String[] USAGE = new String[] {
          "<input> [<input> ...] <output>",
          "where <input> is the source parquet files/directory to be merged",
          "   <output> is the destination parquet file"
  };

  /**
   * Biggest number of input files we can merge.
   */
  private static final int MAX_FILE_NUM = 100;
  private static final long TOO_SMALL_FILE_THRESHOLD = 64 * 1024 * 1024;

  private Configuration conf;

  public MergeCommand() {
    super(2, MAX_FILE_NUM + 1);

    conf = new Configuration();
  }

  @Override
  public String[] getUsageDescription() {
    return USAGE;
  }

  @Override
  public String getCommandDescription() {
    return "Merges multiple Parquet files into one. " +
      "The command doesn't merge row groups, just places one after the other. " +
      "When used to merge many small files, the resulting file will still contain small row groups, " +
      "which usually leads to bad query performance.";
  }

  @Override
  public void execute(CommandLine options) throws Exception {
    // Prepare arguments
    List<String> args = options.getArgList();
    List<Path> inputFiles = getInputFiles(args.subList(0, args.size() - 1));
    Path outputFile = new Path(args.get(args.size() - 1));

    // Merge schema and extraMeta
    FileMetaData mergedMeta = mergedMetadata(inputFiles);
    PrintWriter out = new PrintWriter(Main.out, true);

    // Merge data
    ParquetFileWriter writer = new ParquetFileWriter(conf,
            mergedMeta.getSchema(), outputFile, ParquetFileWriter.Mode.CREATE);
    writer.start();
    boolean tooSmallFilesMerged = false;
    for (Path input: inputFiles) {
      if (input.getFileSystem(conf).getFileStatus(input).getLen() < TOO_SMALL_FILE_THRESHOLD) {
        out.format("Warning: file %s is too small, length: %d\n",
          input,
          input.getFileSystem(conf).getFileStatus(input).getLen());
        tooSmallFilesMerged = true;
      }

      writer.appendFile(HadoopInputFile.fromPath(input, conf));
    }

    if (tooSmallFilesMerged) {
      out.println("Warning: you merged too small files. " +
        "Although the size of the merged file is bigger, it STILL contains small row groups, thus you don't have the advantage of big row groups, " +
        "which usually leads to bad query performance!");
    }
    writer.end(mergedMeta.getKeyValueMetaData());
  }

  private FileMetaData mergedMetadata(List<Path> inputFiles) throws IOException {
    return ParquetFileWriter.mergeMetadataFiles(inputFiles, conf).getFileMetaData();
  }

  /**
   * Get all input files.
   * @param input input files or directory.
   * @return ordered input files.
   */
  private List<Path> getInputFiles(List<String> input) throws IOException {
    List<Path> inputFiles = null;

    if (input.size() == 1) {
      Path p = new Path(input.get(0));
      FileSystem fs = p.getFileSystem(conf);
      FileStatus status = fs.getFileStatus(p);

      if (status.isDir()) {
        inputFiles = getInputFilesFromDirectory(status);
      }
    } else {
      inputFiles = parseInputFiles(input);
    }

    checkParquetFiles(inputFiles);

    return inputFiles;
  }

  /**
   * Check input files basically.
   * ParquetFileReader will throw exception when reading an illegal parquet file.
   *
   * @param inputFiles files to be merged.
   * @throws IOException
   */
  private void checkParquetFiles(List<Path> inputFiles) throws IOException {
    if (inputFiles == null || inputFiles.size() <= 1) {
      throw new IllegalArgumentException("Not enough files to merge");
    }

    for (Path inputFile: inputFiles) {
      FileSystem fs = inputFile.getFileSystem(conf);
      FileStatus status = fs.getFileStatus(inputFile);

      if (status.isDir()) {
        throw new IllegalArgumentException("Illegal parquet file: " + inputFile.toUri());
      }
    }
  }

  /**
   * Get all parquet files under partition directory.
   * @param partitionDir partition directory.
   * @return parquet files to be merged.
   */
  private List<Path> getInputFilesFromDirectory(FileStatus partitionDir) throws IOException {
    FileSystem fs = partitionDir.getPath().getFileSystem(conf);
    FileStatus[] inputFiles = fs.listStatus(partitionDir.getPath(), HiddenFileFilter.INSTANCE);

    List<Path> input = new ArrayList<Path>();
    for (FileStatus f: inputFiles) {
      input.add(f.getPath());
    }
    return input;
  }

  private List<Path> parseInputFiles(List<String> input) {
    List<Path> inputFiles = new ArrayList<Path>();

    for (String name: input) {
      inputFiles.add(new Path(name));
    }

    return inputFiles;
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.column.values.bitpacking;

import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxInt;
import static org.apache.parquet.column.values.bitpacking.BitPacking.createBitPackingReader;

import java.io.IOException;

import org.apache.parquet.bytes.ByteBufferInputStream;
import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.values.ValuesReader;
import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
import org.apache.parquet.io.ParquetDecodingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * a column reader that packs the ints in the number of bits required based on the maximum size.
 */
public class BitPackingValuesReader extends ValuesReader {
  private static final Logger LOG = LoggerFactory.getLogger(BitPackingValuesReader.class);

  private ByteBufferInputStream in;
  private BitPackingReader bitPackingReader;
  private final int bitsPerValue;

  /**
   * @param bound the maximum value stored by this column
   */
  public BitPackingValuesReader(int bound) {
    this.bitsPerValue = getWidthFromMaxInt(bound);
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesReader#readInteger()
   */
  @Override
  public int readInteger() {
    try {
      return bitPackingReader.read();
    } catch (IOException e) {
      throw new ParquetDecodingException(e);
    }
  }

  /**
   * {@inheritDoc}
   * @see org.apache.parquet.column.values.ValuesReader#initFromPage(int, ByteBufferInputStream)
   */
  @Override
  public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException {
    int effectiveBitLength = valueCount * bitsPerValue;
    int length = BytesUtils.paddedByteCountFromBits(effectiveBitLength);
    LOG.debug("reading {} bytes for {} values of size {} bits.", length, valueCount, bitsPerValue);

    this.in = stream.sliceStream(length);
    this.bitPackingReader = createBitPackingReader(bitsPerValue, this.in, valueCount);
    updateNextOffset(length);
  }

  @Override
  public void skip() {
    readInteger();
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.filter;

import org.apache.parquet.column.ColumnReader;

/**
 * Filter which will only materialize a page worth of results.
 */
public final class PagedRecordFilter implements RecordFilter {

  private final long startPos;
  private final long endPos;
  private long currentPos = 0;

  /**
   * Returns builder for creating a paged query.
   * @param startPos The record to start from, numbering starts at 1.
   * @param pageSize The size of the page.
   * @return a paged record filter
   */
  public static final UnboundRecordFilter page( final long startPos, final long pageSize ) {
    return new UnboundRecordFilter() {
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        return new PagedRecordFilter( startPos, pageSize );
      }
    };
  }

  /**
   * Private constructor, use column() instead.
   */
  private PagedRecordFilter(long startPos, long pageSize) {
    this.startPos = startPos;
    this.endPos   = startPos + pageSize;
  }

  /**
   * Keeps track of how many times it is called. Only returns matches when the
   * record number is in the range.
   */
  @Override
  public boolean isMatch() {
    currentPos++;
    return (( currentPos >= startPos ) && ( currentPos < endPos ));
  }

}


/*
 *  Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 */

package org.apache.parquet.io;

import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;

/**
 * Implements read methods required by {@link SeekableInputStream} for generic input streams.
 * <p>
 * Implementations must implement {@link #getPos()} and {@link #seek(long)} and may optionally
 * implement other read methods to improve performance.
 */
public abstract class DelegatingSeekableInputStream extends SeekableInputStream {

  private final int COPY_BUFFER_SIZE = 8192;
  private final byte[] temp = new byte[COPY_BUFFER_SIZE];

  private final InputStream stream;

  public DelegatingSeekableInputStream(InputStream stream) {
    this.stream = stream;
  }

  public InputStream getStream() {
    return stream;
  }

  @Override
  public void close() throws IOException {
    stream.close();
  }

  @Override
  public abstract long getPos() throws IOException;

  @Override
  public abstract void seek(long newPos) throws IOException;

  @Override
  public int read() throws IOException {
    return stream.read();
  }

  @Override
  public int read(byte[] b, int off, int len) throws IOException {
    return stream.read(b, off, len);
  }

  @Override
  public void readFully(byte[] bytes) throws IOException {
    readFully(stream, bytes, 0, bytes.length);
  }

  @Override
  public void readFully(byte[] bytes, int start, int len) throws IOException {
    readFully(stream, bytes, start, len);
  }

  @Override
  public int read(ByteBuffer buf) throws IOException {
    if (buf.hasArray()) {
      return readHeapBuffer(stream, buf);
    } else {
      return readDirectBuffer(stream, buf, temp);
    }
  }

  @Override
  public void readFully(ByteBuffer buf) throws IOException {
    if (buf.hasArray()) {
      readFullyHeapBuffer(stream, buf);
    } else {
      readFullyDirectBuffer(stream, buf, temp);
    }
  }

  // Visible for testing
  static void readFully(InputStream f, byte[] bytes, int start, int len) throws IOException {
    int offset = start;
    int remaining = len;
    while (remaining > 0) {
      int bytesRead = f.read(bytes, offset, remaining);
      if (bytesRead < 0) {
        throw new EOFException(
            "Reached the end of stream with " + remaining + " bytes left to read");
      }

      remaining -= bytesRead;
      offset += bytesRead;
    }
  }

  // Visible for testing
  static int readHeapBuffer(InputStream f, ByteBuffer buf) throws IOException {
    int bytesRead = f.read(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining());
    if (bytesRead < 0) {
      // if this resulted in EOF, don't update position
      return bytesRead;
    } else {
      buf.position(buf.position() + bytesRead);
      return bytesRead;
    }
  }

  // Visible for testing
  static void readFullyHeapBuffer(InputStream f, ByteBuffer buf) throws IOException {
    readFully(f, buf.array(), buf.arrayOffset() + buf.position(), buf.remaining());
    buf.position(buf.limit());
  }

  // Visible for testing
  static int readDirectBuffer(InputStream f, ByteBuffer buf, byte[] temp) throws IOException {
    // copy all the bytes that return immediately, stopping at the first
    // read that doesn't return a full buffer.
    int nextReadLength = Math.min(buf.remaining(), temp.length);
    int totalBytesRead = 0;
    int bytesRead;

    while ((bytesRead = f.read(temp, 0, nextReadLength)) == temp.length) {
      buf.put(temp);
      totalBytesRead += bytesRead;
      nextReadLength = Math.min(buf.remaining(), temp.length);
    }

    if (bytesRead < 0) {
      // return -1 if nothing was read
      return totalBytesRead == 0 ? -1 : totalBytesRead;
    } else {
      // copy the last partial buffer
      buf.put(temp, 0, bytesRead);
      totalBytesRead += bytesRead;
      return totalBytesRead;
    }
  }

  // Visible for testing
  static void readFullyDirectBuffer(InputStream f, ByteBuffer buf, byte[] temp) throws IOException {
    int nextReadLength = Math.min(buf.remaining(), temp.length);
    int bytesRead = 0;

    while (nextReadLength > 0 && (bytesRead = f.read(temp, 0, nextReadLength)) >= 0) {
      buf.put(temp, 0, bytesRead);
      nextReadLength = Math.min(buf.remaining(), temp.length);
    }

    if (bytesRead < 0 && buf.remaining() > 0) {
      throw new EOFException(
          "Reached the end of stream with " + buf.remaining() + " bytes left to read");
    }
  }
}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.format.event;

import static java.util.Collections.unmodifiableMap;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.TFieldIdEnum;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TProtocolUtil;

import org.apache.parquet.format.event.Consumers.Consumer;
import org.apache.parquet.format.event.TypedConsumer.ListConsumer;
import org.apache.parquet.format.event.TypedConsumer.StructConsumer;

/**
 * Entry point for reading thrift in a streaming fashion
 */
public class Consumers {

  /**
   * To consume objects coming from a DelegatingFieldConsumer
   *
   * @param <T> the type of consumed objects
   */
  public static interface Consumer<T> {
    void consume(T t);
  }

  /**
   * Delegates reading the field to TypedConsumers.
   * There is one TypedConsumer per thrift type.
   * use {@link #onField(TFieldIdEnum, TypedConsumer)} et al. to consume specific thrift fields.
   * @see Consumers#fieldConsumer()
   */
  public static class DelegatingFieldConsumer implements FieldConsumer {

    private final Map<Short, TypedConsumer> contexts;
    private final FieldConsumer defaultFieldEventConsumer;

    private DelegatingFieldConsumer(FieldConsumer defaultFieldEventConsumer, Map<Short, TypedConsumer> contexts) {
      this.defaultFieldEventConsumer = defaultFieldEventConsumer;
      this.contexts = unmodifiableMap(contexts);
    }

    private DelegatingFieldConsumer() {
      this(new SkippingFieldConsumer());
    }

    private DelegatingFieldConsumer(FieldConsumer defaultFieldEventConsumer) {
      this(defaultFieldEventConsumer, Collections.<Short, TypedConsumer>emptyMap());
    }

    public DelegatingFieldConsumer onField(TFieldIdEnum e, TypedConsumer typedConsumer) {
      Map<Short, TypedConsumer> newContexts = new HashMap<Short, TypedConsumer>(contexts);
      newContexts.put(e.getThriftFieldId(), typedConsumer);
      return new DelegatingFieldConsumer(defaultFieldEventConsumer, newContexts);
    }

    @Override
    public void consumeField(
        TProtocol protocol, EventBasedThriftReader reader,
        short id, byte type) throws TException {
      TypedConsumer delegate = contexts.get(id);
      if (delegate != null) {
        delegate.read(protocol, reader, type);
      } else {
        defaultFieldEventConsumer.consumeField(protocol, reader, id, type);
      }
    }
  }

  /**
   * call onField on the resulting DelegatingFieldConsumer to handle individual fields
   * @return a new DelegatingFieldConsumer
   */
  public static DelegatingFieldConsumer fieldConsumer() {
    return new DelegatingFieldConsumer();
  }

  /**
   * To consume a list of elements
   * @param c the class of the list content
   * @param consumer the consumer that will receive the list
   * @param <T> the type of the list content
   * @return a ListConsumer that can be passed to the DelegatingFieldConsumer
   */
  public static <T extends TBase<T,? extends TFieldIdEnum>> ListConsumer listOf(Class<T> c, final Consumer<List<T>> consumer) {
    class ListConsumer implements Consumer<T> {
      List<T> list;
      @Override
      public void consume(T t) {
        list.add(t);
      }
    }
    final ListConsumer co = new ListConsumer();
    return new DelegatingListElementsConsumer(struct(c, co)) {
      @Override
      public void consumeList(TProtocol protocol,
          EventBasedThriftReader reader, TList tList) throws TException {
        co.list = new ArrayList<T>();
        super.consumeList(protocol, reader, tList);
        consumer.consume(co.list);
      }
    };
  }

  /**
   * To consume list elements one by one
   * @param consumer the consumer that will read the elements
   * @return a ListConsumer that can be passed to the DelegatingFieldConsumer
   */
  public static ListConsumer listElementsOf(TypedConsumer consumer) {
    return new DelegatingListElementsConsumer(consumer);
  }

  public static <T extends TBase<T,? extends TFieldIdEnum>> StructConsumer struct(final Class<T> c, final Consumer<T> consumer) {
    return new TBaseStructConsumer<T>(c, consumer);
  }
}

class SkippingFieldConsumer implements FieldConsumer {
  @Override
  public void consumeField(TProtocol protocol, EventBasedThriftReader reader, short id, byte type) throws TException {
    TProtocolUtil.skip(protocol, type);
  }
}

class DelegatingListElementsConsumer extends ListConsumer {

  private TypedConsumer elementConsumer;

  protected DelegatingListElementsConsumer(TypedConsumer consumer) {
    this.elementConsumer = consumer;
  }

  @Override
  public void consumeElement(TProtocol protocol, EventBasedThriftReader reader, byte elemType) throws TException {
    elementConsumer.read(protocol, reader, elemType);
  }
}
class TBaseStructConsumer<T extends TBase<T, ? extends TFieldIdEnum>> extends StructConsumer {

  private final Class<T> c;
  private Consumer<T> consumer;

  public TBaseStructConsumer(Class<T> c, Consumer<T> consumer) {
    this.c = c;
    this.consumer = consumer;
  }

  @Override
  public void consumeStruct(TProtocol protocol, EventBasedThriftReader reader) throws TException {
    T o = newObject();
    o.read(protocol);
    consumer.consume(o);
  }

  protected T newObject() {
    try {
      return c.newInstance();
    } catch (InstantiationException e) {
      throw new RuntimeException(c.getName(), e);
    } catch (IllegalAccessException e) {
      throw new RuntimeException(c.getName(), e);
    }
  }

}


/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.hive.internal;

import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapred.JobConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Hive 0.10 implementation of {@link org.apache.parquet.hive.HiveBinding HiveBinding}.
 * This class is a renamed version of
 * ManageJobConfig class.
 */
public class Hive010Binding extends AbstractHiveBinding {
  private static final Logger LOG = LoggerFactory.getLogger(Hive010Binding.class);
  private final Map<String, PartitionDesc> pathToPartitionInfo =
      new LinkedHashMap<String, PartitionDesc>();
  /**
   * MapredWork is the Hive object which describes input files,
   * columns projections, and filters.
   */
  private MapredWork mrwork;

  /**
   * Initialize the mrwork variable in order to get all the partition and start to update the jobconf
   *
   * @param job
   */
  private void init(final JobConf job) {
    final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN);
    if (mrwork == null && plan != null && plan.length() > 0) {
      mrwork = Utilities.getMapRedWork(job);
      pathToPartitionInfo.clear();
      for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) {
        pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue());
      }
    }
  }

  private void pushProjectionsAndFilters(final JobConf jobConf,
      final String splitPath, final String splitPathWithNoSchema) {

    if (mrwork == null) {
      LOG.debug("Not pushing projections and filters because MapredWork is null");
      return;
    } else if (mrwork.getPathToAliases() == null) {
      LOG.debug("Not pushing projections and filters because pathToAliases is null");
      return;
    }

    final ArrayList<String> aliases = new ArrayList<String>();
    final Iterator<Entry<String, ArrayList<String>>> iterator = mrwork.getPathToAliases().entrySet().iterator();

    while (iterator.hasNext()) {
      final Entry<String, ArrayList<String>> entry = iterator.next();
      final String key = new Path(entry.getKey()).toUri().getPath();

      if (splitPath.equals(key) || splitPathWithNoSchema.equals(key)) {
        final ArrayList<String> list = entry.getValue();
        for (final String val : list) {
          aliases.add(val);
        }
      }
    }

    for (final String alias : aliases) {
      final Operator<? extends Serializable> op = mrwork.getAliasToWork().get(
              alias);
      if (op != null && op instanceof TableScanOperator) {
        final TableScanOperator tableScan = (TableScanOperator) op;

        // push down projections
        final ArrayList<Integer> list = tableScan.getNeededColumnIDs();

        if (list != null) {
          ColumnProjectionUtils.appendReadColumnIDs(jobConf, list);
        } else {
          ColumnProjectionUtils.setFullyReadColumns(jobConf);
        }

        pushFilters(jobConf, tableScan);
      }
    }
  }

  private void pushFilters(final JobConf jobConf, final TableScanOperator tableScan) {

    final TableScanDesc scanDesc = tableScan.getConf();
    if (scanDesc == null) {
      LOG.debug("Not pushing filters because TableScanDesc is null");
      return;
    }

    // construct column name list for reference by filter push down
    Utilities.setColumnNameList(jobConf, tableScan);

    // push down filters
    final ExprNodeDesc filterExpr = scanDesc.getFilterExpr();
    if (filterExpr == null) {
      LOG.debug("Not pushing filters because FilterExpr is null");
      return;
    }

    final String filterText = filterExpr.getExprString();
    final String filterExprSerialized = Utilities.serializeExpression(filterExpr);
    jobConf.set(
            TableScanDesc.FILTER_TEXT_CONF_STR,
            filterText);
    jobConf.set(
            TableScanDesc.FILTER_EXPR_CONF_STR,
            filterExprSerialized);
  }

  /**
   * {@inheritDoc}
   */
  @Override
  public JobConf pushProjectionsAndFilters(JobConf jobConf, Path path)
      throws IOException {
    init(jobConf);
    final JobConf cloneJobConf = new JobConf(jobConf);
    final PartitionDesc part = pathToPartitionInfo.get(path.toString());

    if ((part != null) && (part.getTableDesc() != null)) {
      Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
    }

    pushProjectionsAndFilters(cloneJobConf, path.toString(), path.toUri().toString());
    return cloneJobConf;
  }

}