/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine;

import java.io.DataInputStream;
import java.io.IOException;

public class SnapshotPopulatedOrdinalsReader {

    /**
     * Read populated ordinals as a bit set from a stream, and notify a listener for each populated ordinal.
     *
     * @param dis the data input stream
     * @param listeners the type state listeners
     * @throws IOException if the ordinals cannot be read
     * @author dkoszewnik
     */
    public static void readOrdinals(DataInputStream dis, HollowTypeStateListener[] listeners) throws IOException {
        int numLongs = dis.readInt();

        int currentOrdinal = 0;

        for(int i=0;i<numLongs;i++) {
            long l = dis.readLong();
            notifyPopulatedOrdinals(l, currentOrdinal, listeners);
            currentOrdinal += 64;
        }
    }

    private static void notifyPopulatedOrdinals(long l, int ordinal, HollowTypeStateListener[] listeners) {
        if(l == 0)
            return;

        int stopOrdinal = ordinal + 64;

        while(ordinal < stopOrdinal) {
            long mask = 1L << ordinal;
            if((l & mask) != 0) {
                for(int i=0; i<listeners.length; i++) {
                    listeners[i].addedOrdinal(ordinal);
                }
            }
            ordinal++;
        }
    }

    public static void discardOrdinals(DataInputStream dis) throws IOException {
        long numLongs = dis.readInt();
        long bytesToSkip = numLongs * 8;

        while(bytesToSkip > 0)
            bytesToSkip -= dis.skip(bytesToSkip);
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeStateListener;
import java.util.BitSet;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReferenceArray;

/**
 * Create hollow integer set for sparse non-negative {@literal &} unique integer values referenced by fieldPath in a type based on a predicate.
 */
public class HollowSparseIntegerSet implements HollowTypeStateListener {

    private final HollowReadStateEngine readStateEngine;
    private final String type;
    private final FieldPath fieldPath;
    private final IndexPredicate predicate;

    protected volatile SparseBitSet sparseBitSetVolatile;

    private Set<Integer> valuesToSet;
    private Set<Integer> valuesToClear;
    private int maxValueToSet;

    public interface IndexPredicate {
        boolean shouldIndex(int ordinal);
    }

    private final static IndexPredicate DEFAULT_PREDICATE = new IndexPredicate() {
        @Override
        public boolean shouldIndex(int ordinal) {
            return true;
        }
    };

    /**
     * Create a index for integer values pointed by the given field path.
     *
     * @param readStateEngine the read state
     * @param type the type name
     * @param fieldPath the field path
     */
    public HollowSparseIntegerSet(HollowReadStateEngine readStateEngine, String type, String fieldPath) {
        this(readStateEngine, type, fieldPath, DEFAULT_PREDICATE);
    }

    /**
     * Create a index for integer values based on the given IndexPredicate.
     *
     * @param readStateEngine HollowReadStateEngine to read data set.
     * @param type            type which contains the path to integer values for indexing.
     * @param fieldPath       path to the integer values
     * @param predicate       implementation of IndexPredicate, indicating if the record passes the condition for indexing.
     */
    public HollowSparseIntegerSet(HollowReadStateEngine readStateEngine, String type, String fieldPath, IndexPredicate predicate) {

        // check arguments
        if (readStateEngine == null) throw new IllegalArgumentException("Read state engine cannot be null");
        if (type == null) throw new IllegalArgumentException("type cannot be null");
        if (fieldPath == null || fieldPath.isEmpty())
            throw new IllegalArgumentException("fieldPath cannot be null or empty");

        this.readStateEngine = readStateEngine;
        this.type = type;
        this.fieldPath = new FieldPath(readStateEngine, type, fieldPath);
        this.predicate = predicate;
        this.valuesToSet = new HashSet<>();
        this.valuesToClear = new HashSet<>();
        build();
    }

    protected void build() {

        // initialize an instance of SparseBitSet
        initSet(Integer.MAX_VALUE);

        // iterate through all populated ordinals for the type to set the values based on predicate
        BitSet typeBitSet = readStateEngine.getTypeState(type).getPopulatedOrdinals();
        int ordinal = typeBitSet.nextSetBit(0);
        while (ordinal != -1) {
            set(ordinal);
            ordinal = typeBitSet.nextSetBit(ordinal + 1);
        }

        // run compaction
        compact();
    }

    protected void initSet(int maxValue) {
        sparseBitSetVolatile = new SparseBitSet(maxValue);
    }

    protected void set(int ordinal) {
        if (predicate.shouldIndex(ordinal)) {
            Object[] values = fieldPath.findValues(ordinal);
            if (values != null && values.length > 0) {
                SparseBitSet bitSet = sparseBitSetVolatile;
                for (Object value : values) {
                    bitSet.set((int) value);
                }
            }
        }
    }

    protected void compact() {
        SparseBitSet current = sparseBitSetVolatile;
        SparseBitSet compactedSet = SparseBitSet.compact(current);
        sparseBitSetVolatile = compactedSet;
    }

    /**
     * Check if the given value is contained in the set (or if the given value satisfies the predicate condition.)
     *
     * @param i the integer value
     * @return {@code true} if the value is present
     */
    public boolean get(int i) {
        SparseBitSet current;
        boolean result;
        do {
            current = sparseBitSetVolatile;
            result = current.get(i);
        } while (current != sparseBitSetVolatile);
        return result;
    }

    /**
     * Estimate the total number of bits used to represent the integer set.
     *
     * @return Calculates the total number of bits used by longs in underlying data structure.
     */
    public long size() {
        SparseBitSet current;
        long size;
        do {
            current = sparseBitSetVolatile;
            size = current.estimateBitsUsed();
        } while (current != sparseBitSetVolatile);
        return size;
    }

    /**
     * @return the total number of integers added to the set.
     */
    public int cardinality() {
        SparseBitSet current;
        int cardinality;
        do {
            current = sparseBitSetVolatile;
            cardinality = current.cardinality();
        } while (current != sparseBitSetVolatile);
        return cardinality;
    }

    /**
     * Use this method to keep the index updated with delta changes on the read state engine.
     * Remember to call detachFromDeltaUpdates to stop the delta changes.
     * NOTE: Each delta updates creates a new prefix index and swaps the new with current.
     */
    public void listenForDeltaUpdates() {
        readStateEngine.getTypeState(type).addListener(this);
    }

    /**
     * Stop delta updates for this index.
     */
    public void detachFromDeltaUpdates() {
        readStateEngine.getTypeState(type).removeListener(this);
    }

    @Override
    public void beginUpdate() {
        valuesToSet.clear();
        valuesToClear.clear();
        maxValueToSet = -1;
    }

    @Override
    public void addedOrdinal(int ordinal) {
        if (predicate.shouldIndex(ordinal)) {
            Object[] values = fieldPath.findValues(ordinal);
            for (Object value : values) {
                valuesToSet.add((int) value);
                if (maxValueToSet < (int) value) maxValueToSet = (int) value;
            }
        }
    }

    @Override
    public void removedOrdinal(int ordinal) {
        Object[] values = fieldPath.findValues(ordinal);
        for (Object value : values)
            valuesToClear.add((int) value);
    }

    @Override
    public void endUpdate() {
        boolean didSomeWork = false;
        SparseBitSet updated = sparseBitSetVolatile;
        // first check if the max value among the new values to be added is more than the max value of the existing sparse bit set.
        if (valuesToSet.size() > 0 && maxValueToSet > updated.findMaxValue()) {
            updated = SparseBitSet.resize(updated, maxValueToSet);
            didSomeWork = true;
        }

        // when applying delta, check for duplicates, increment counts if duplicate values are found else set them
        for (int value : valuesToSet) {
            updated.set(value);
        }

        // first clear all the values that are meant to be cleared
        for (int value : valuesToClear) {
            updated.clear(value);
        }

        if (didSomeWork) {
            sparseBitSetVolatile = updated;
        }
    }

    /**
     * This implementation is motivated from several ideas to get a compact sparse set.
     * When using a a bucket of BitSet, problems
     * - smaller sizes of BitSet are not useful, since null references are themselves 64/32 bit references.
     * - larger sizes of BitSet for truly sparse integers, has overhead of too many zeroes in one BitSet.
     *

     * The idea is to only store longs in bb that have non-zero values where bucket sizes are longs. Bucket size of 64 longs are convenient when using mod operations.
     *

     * Each bit in long value in indices array, indicates if a long value is initialized. 64 bits would point to 64 long values ( 1 bucket ).
     * Each bucket could contain 1-64 longs, we only hold non-zero long values in bucket.
     */
    static class SparseBitSet {

        // shift used to determine which bucket
        private static final int BUCKET_SHIFT = 12;
        // shift used to determine which Long value to use in bucket.
        private static final int LONG_SHIFT = 6;

        private final int maxValue;
        private final AtomicReferenceArray<Bucket> buckets;

        private static class Bucket {
            private long idx;
            private long[] longs;

            private Bucket(long idx, long[] longs) {
                this.idx = idx;
                this.longs = longs;
            }
        }

        SparseBitSet(int maxValue) {
            int totalBuckets = maxValue >>> BUCKET_SHIFT;
            this.maxValue = maxValue;
            this.buckets = new AtomicReferenceArray<>(totalBuckets + 1);
        }

        private SparseBitSet(int maxValue, AtomicReferenceArray<Bucket> buckets) {
            this.maxValue = maxValue;
            this.buckets = buckets;
        }

        private static int getIndex(int i) {
            // logical right shift
            return i >>> BUCKET_SHIFT;
        }

        /**
         * This method returns the number of Longs initialized from LSB to the given bitInIndex.
         * For example longAtIndex (64 bits) = 00...1001 and bitInIndex (3rd bit is set) 000...100
         * then this method will return 1 since only one bit is set in longAtIndex to the right of bitInIndex
         *
         * @param longAtIndex
         * @param bitInIndex
         * @return
         */
        private static int getOffset(long longAtIndex, long bitInIndex) {
            // set all bits to one before the bit that is set in bitInIndex
            // example : 000...0100 will become 000...011
            long setAllOnesBeforeBitInIndex = bitInIndex - 1;
            long offset = (longAtIndex & setAllOnesBeforeBitInIndex);
            return Long.bitCount(offset);
        }

        boolean get(int i) {
            if (i > maxValue || i < 0)
                return false;

            int index = getIndex(i);
            Bucket currentBucket = buckets.get(index);
            if (currentBucket == null) return false;

            long currentLongAtIndex = currentBucket.idx;
            long[] longs = currentBucket.longs;

            // find which bit in index will point to the long in bb
            long whichLong = i >>> LONG_SHIFT;
            long bitInIndex = 1L << whichLong;// whichLong % 64

            long isLongInitialized = (currentLongAtIndex & bitInIndex);
            if (isLongInitialized == 0) return false;

            int offset = getOffset(currentLongAtIndex, bitInIndex);
            long value = longs[offset];
            long whichBitInLong = 1L < maxValue)
                throw new IllegalArgumentException("Max value initialized is " + maxValue + " given value is " + i);
            if (i < 0) throw new IllegalArgumentException("Cannot index negative numbers");

            // find which bucket
            int index = getIndex(i);

            // find which bit in index will point to the long in bb
            long whichLong = i >>> LONG_SHIFT;
            long bitInIndex = 1L << whichLong;// whichLong % 64
            long whichBitInLong = 1L << i;// i % 64

            while (true) {

                long longAtIndex = 0;
                long[] longs = null;

                Bucket currentBucket = buckets.get(index);
                if (currentBucket != null) {
                    longAtIndex = currentBucket.idx;
                    longs = currentBucket.longs.clone();
                }

                boolean isLongInitialized = (longAtIndex & bitInIndex) != 0;
                if (isLongInitialized) {

                    // if a long value is set, the find the correct offset to determine which long in longs to use.
                    int offset = getOffset(longAtIndex, bitInIndex);
                    longs[offset] |= whichBitInLong;// or preserves previous set operations in this long.

                } else if (longAtIndex == 0) {

                    // first set that bit in idx for that bucket, and assign a new long[]
                    longAtIndex = bitInIndex;
                    longs = new long[]{whichBitInLong};

                } else {

                    // update long value at index
                    longAtIndex |= bitInIndex;

                    // find offset
                    int offset = getOffset(longAtIndex, bitInIndex);

                    int oldLongsLen = longs.length;
                    long[] newLongs = new long[oldLongsLen + 1];

                    // if offset is 2 means 3 longs are needed starting from 0
                    // if current longs length is 2 (0,1) then append third long at end
                    // if current longs length is greater than offset, then insert long 0 -> (offset - 1), new long, offset to (length -1)
                    if (offset >= oldLongsLen) {
                        // append new long at end
                        int it;
                        for (it = 0; it < oldLongsLen; it++)
                            newLongs[it] = longs[it];
                        newLongs[it] = whichBitInLong;
                    } else {
                        // insert new long in between
                        int it;
                        for (it = 0; it < offset; it++)
                            newLongs[it] = longs[it];
                        newLongs[offset] = whichBitInLong;
                        for (it = offset; it < oldLongsLen; it++)
                            newLongs[it + 1] = longs[it];
                    }
                    longs = newLongs;
                }

                Bucket newBucket = new Bucket(longAtIndex, longs);
                if (buckets.compareAndSet(index, currentBucket, newBucket))
                    break;
            }
        }

        // thread-safe
        void clear(int i) {
            if (i > maxValue || i < 0) return;

            int index = getIndex(i);

            while (true) {
                Bucket currentBucket = buckets.get(index);
                if (currentBucket == null) return;
                long longAtIndex = currentBucket.idx;
                long[] longs = currentBucket.longs.clone();

                // find which bit in index will point to the long in bb
                long whichLong = i >>> LONG_SHIFT;
                long bitInIndex = 1L << whichLong;// whichLong % 64
                long whichBitInLong = 1L << i;// i % 64

                long isLongInitialized = (longAtIndex & bitInIndex);
                if (isLongInitialized == 0) return;

                int offset = getOffset(longAtIndex, bitInIndex);
                long value = longs[offset];

                // unset whichBitInIndex in value
                // to clear 3rd bit (00100 whichBitInLong) in 00101(value), & with 11011 to get 00001
                long updatedValue = value & ~whichBitInLong;
                boolean isBucketEmpty = false;
                if (updatedValue != 0) {
                    longs[offset] = updatedValue;
                } else {

                    // if updatedValue is 0, then update the bucket removing that long
                    int oldLongsLen = longs.length;
                    // if only one long was initialized in the bucket, then make the reference null, indexAtLong 0
                    if (oldLongsLen == 1) {
                        longs = null;
                        longAtIndex = 0;
                        isBucketEmpty = true;
                    } else {
                        // copy everything over, except the long at the given offset,

                        long[] newLongs = new long[oldLongsLen - 1];

                        int it;
                        for (it = 0; it < offset; it++)
                            newLongs[it] = longs[it];
                        it++;
                        while (it < oldLongsLen) {
                            newLongs[it - 1] = longs[it];
                            it++;
                        }

                        longs = newLongs;
                        longAtIndex &= ~bitInIndex;
                    }
                }
                Bucket updatedBucket = null;
                if (!isBucketEmpty) updatedBucket = new Bucket(longAtIndex, longs);
                if (buckets.compareAndSet(index, currentBucket, updatedBucket))
                    break;

            }
        }

        int findMaxValue() {
            // find the last index that is initialized
            int index = buckets.length() - 1;
            while (index >= 0) {
                if (buckets.get(index) != null) break;
                index--;
            }

            // if no buckets are initialized, then return -1 ( meaning set is empty)
            if (index < 0) return -1;

            // find the highest bit in indexAtLong to see which is last long init in bucket
            int highestBitSetInIndexAtLong = 63 - Long.numberOfLeadingZeros(Long.highestOneBit(buckets.get(index).idx));

            long[] longs = buckets.get(index).longs;
            long value = longs[longs.length - 1];
            long highestBitSetInLong = 63 - Long.numberOfLeadingZeros(Long.highestOneBit(value));

            return (int) ((index << BUCKET_SHIFT) + (highestBitSetInIndexAtLong << 6) + highestBitSetInLong);
        }

        int cardinality() {
            int cardinality = 0;
            int index = 0;
            while (index < buckets.length()) {

                if (buckets.get(index) != null) {
                    long[] longs = buckets.get(index).longs;
                    for (long value : longs)
                        cardinality += Long.bitCount(value);
                }
                index++;
            }
            return cardinality;
        }

        long estimateBitsUsed() {
            long longsUsed = 0;
            long idxCounts = 0;

            int index = 0;
            while (index < buckets.length()) {
                if (buckets.get(index) != null) {
                    idxCounts++;
                    longsUsed += buckets.get(index).longs.length;
                }
                index++;
            }

            // total bits used
            long bitsUsedByArrayPointers = buckets.length() * 64;
            long bitsUsedByIdx = idxCounts * 64;
            long bitsUsedByLongs = longsUsed * 64;

            return bitsUsedByArrayPointers + bitsUsedByIdx + bitsUsedByLongs;
        }

        /**
         * * Use this method to compact an existing SparseBitSet. Note any attempts to add a new value greater than the max value will result in exception.
         *
         * @param sparseBitSet
         * @return new SparseBitSet that is compact, does not hold null references beyond the max int value added in the given input.
         */
        static SparseBitSet compact(SparseBitSet sparseBitSet) {
            int maxValueAdded = sparseBitSet.findMaxValue();
            // if the given set is empty then compact the sparseBitSet to have only 1 bucket i.e. 64 longs
            if (maxValueAdded < 0) {
                maxValueAdded = (1 << BUCKET_SHIFT) - 1;
            }
            int indexForMaxValueAdded = getIndex(maxValueAdded);
            int newLength = indexForMaxValueAdded + 1;
            return copyWithNewLength(sparseBitSet, newLength, newLength, maxValueAdded);
        }

        static SparseBitSet resize(SparseBitSet sparseBitSet, int newMaxValue) {
            if (sparseBitSet.findMaxValue() < newMaxValue) {
                int indexForNewMaxValue = getIndex(newMaxValue);
                int newLength = indexForNewMaxValue + 1;
                return copyWithNewLength(sparseBitSet, newLength, sparseBitSet.buckets.length(), newMaxValue);
            }
            return sparseBitSet;
        }

        private static SparseBitSet copyWithNewLength(SparseBitSet sparseBitSet, int newLength, int lengthToClone, int newMaxValue) {
            AtomicReferenceArray<Bucket> compactBuckets = new AtomicReferenceArray<Bucket>(newLength);
            for (int i = 0; i < lengthToClone; i++) {
                if (sparseBitSet.buckets.get(i) != null) compactBuckets.set(i, sparseBitSet.buckets.get(i));
            }
            return new SparseBitSet(newMaxValue, compactBuckets);
        }

    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine;

import com.netflix.hollow.core.HollowBlobHeader;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.read.engine.list.HollowListTypeReadState;
import com.netflix.hollow.core.read.engine.map.HollowMapTypeReadState;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.engine.set.HollowSetTypeReadState;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.schema.HollowListSchema;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSetSchema;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.TreeSet;
import java.util.logging.Logger;

/**
 * A HollowBlobReader is used to populate and update data in a {@link HollowReadStateEngine}, via the consumption
 * of snapshot and delta blobs.
 */
public class HollowBlobReader {

    private final Logger log = Logger.getLogger(HollowBlobReader.class.getName());
    private final HollowReadStateEngine stateEngine;
    private final HollowBlobHeaderReader headerReader;

    public HollowBlobReader(HollowReadStateEngine stateEngine) {
        this(stateEngine, new HollowBlobHeaderReader());
    }

    public HollowBlobReader(HollowReadStateEngine stateEngine, HollowBlobHeaderReader headerReader) {
        this.stateEngine = stateEngine;
        this.headerReader = headerReader;
    }

    /**
     * Initialize the state engine using a snapshot blob from the provided InputStream.
     *
     * @param is the input stream to read the snapshot from
     * @throws IOException if the snapshot could not be read
     */
    public void readSnapshot(InputStream is) throws IOException {
        readSnapshot(is, new HollowFilterConfig(true));
    }

    /**
     * Initialize the state engine using a snapshot blob from the provided InputStream.
     *

     * Apply the provided {@link HollowFilterConfig} to the state.
     *
     * @param is the input stream to read the snaptshot
     * @param filter the filtering configuration to filter the snapshot
     * @throws IOException if the snapshot could not be read
     */
    public void readSnapshot(InputStream is, HollowFilterConfig filter) throws IOException {
        HollowBlobHeader header = readHeader(is, false);

        notifyBeginUpdate();

        long startTime = System.currentTimeMillis();

        DataInputStream dis = new DataInputStream(is);

        int numStates = VarInt.readVInt(dis);

        Collection<String> typeNames = new TreeSet<String>();
        for(int i=0;i<numStates;i++) {
            String typeName = readTypeStateSnapshot(dis, header, filter);
            typeNames.add(typeName);
        }

        stateEngine.wireTypeStatesToSchemas();

        long endTime = System.currentTimeMillis();

        log.info("SNAPSHOT COMPLETED IN " + (endTime - startTime) + "ms");
        log.info("TYPES: " + typeNames);

        notifyEndUpdate();

        stateEngine.afterInitialization();
    }

    /**
     * Update the state engine using a delta (or reverse delta) blob from the provided InputStream.
     *

     * If a {@link HollowFilterConfig} was applied at the time the {@link HollowReadStateEngine} was initialized
     * with a snapshot, it will continue to be in effect after the state is updated.
     *
     * @param is the input stream to read the delta from
     * @throws IOException if the delta could not be applied
     */
    public void applyDelta(InputStream is) throws IOException {
        HollowBlobHeader header = readHeader(is, true);
        notifyBeginUpdate();

        long startTime = System.currentTimeMillis();

        DataInputStream dis = new DataInputStream(is);

        int numStates = VarInt.readVInt(dis);

        Collection<String> typeNames = new TreeSet<String>();
        for(int i=0;i<numStates;i++) {
            String typeName = readTypeStateDelta(dis, header);
            typeNames.add(typeName);
            stateEngine.getMemoryRecycler().swap();
        }

        long endTime = System.currentTimeMillis();

        log.info("DELTA COMPLETED IN " + (endTime - startTime) + "ms");
        log.info("TYPES: " + typeNames);

        notifyEndUpdate();

    }

    private HollowBlobHeader readHeader(InputStream is, boolean isDelta) throws IOException {
        HollowBlobHeader header = headerReader.readHeader(is);

        if(isDelta && header.getOriginRandomizedTag() != stateEngine.getCurrentRandomizedTag())
            throw new IOException("Attempting to apply a delta to a state from which it was not originated!");

        stateEngine.setCurrentRandomizedTag(header.getDestinationRandomizedTag());
        stateEngine.setHeaderTags(header.getHeaderTags());
        return header;
    }

    private void notifyBeginUpdate() {
        for(HollowTypeReadState typeState : stateEngine.getTypeStates()) {
            for(HollowTypeStateListener listener : typeState.getListeners()) {
                listener.beginUpdate();
            }
        }
    }

    private void notifyEndUpdate() {
        for(HollowTypeReadState typeState : stateEngine.getTypeStates()) {
            for(HollowTypeStateListener listener : typeState.getListeners()) {
                listener.endUpdate();
            }
        }
    }

    private String readTypeStateSnapshot(DataInputStream is, HollowBlobHeader header, HollowFilterConfig filter) throws IOException {
        HollowSchema schema = HollowSchema.readFrom(is);

        int numShards = readNumShards(is);

        if(schema instanceof HollowObjectSchema) {
            if(!filter.doesIncludeType(schema.getName())) {
                HollowObjectTypeReadState.discardSnapshot(is, (HollowObjectSchema)schema, numShards);
            } else {
                HollowObjectSchema unfilteredSchema = (HollowObjectSchema)schema;
                HollowObjectSchema filteredSchema = unfilteredSchema.filterSchema(filter);
                populateTypeStateSnapshot(is, new HollowObjectTypeReadState(stateEngine, filteredSchema, unfilteredSchema, numShards));
            }
        } else if (schema instanceof HollowListSchema) {
            if(!filter.doesIncludeType(schema.getName())) {
                HollowListTypeReadState.discardSnapshot(is, numShards);
            } else {
                populateTypeStateSnapshot(is, new HollowListTypeReadState(stateEngine, (HollowListSchema)schema, numShards));
            }
        } else if(schema instanceof HollowSetSchema) {
            if(!filter.doesIncludeType(schema.getName())) {
                HollowSetTypeReadState.discardSnapshot(is, numShards);
            } else {
                populateTypeStateSnapshot(is, new HollowSetTypeReadState(stateEngine, (HollowSetSchema)schema, numShards));
            }
        } else if(schema instanceof HollowMapSchema) {
            if(!filter.doesIncludeType(schema.getName())) {
                HollowMapTypeReadState.discardSnapshot(is, numShards);
            } else {
                populateTypeStateSnapshot(is, new HollowMapTypeReadState(stateEngine, (HollowMapSchema)schema, numShards));
            }
        }

        return schema.getName();
    }

    private void populateTypeStateSnapshot(DataInputStream is, HollowTypeReadState typeState) throws IOException {
        stateEngine.addTypeState(typeState);
        typeState.readSnapshot(is, stateEngine.getMemoryRecycler());
    }

    private String readTypeStateDelta(DataInputStream is, HollowBlobHeader header) throws IOException {
        HollowSchema schema = HollowSchema.readFrom(is);

        int numShards = readNumShards(is);

        HollowTypeReadState typeState = stateEngine.getTypeState(schema.getName());
        if(typeState != null) {
            typeState.applyDelta(is, schema, stateEngine.getMemoryRecycler());
        } else {
            discardDelta(is, schema, numShards);
        }

        return schema.getName();
    }

    private int readNumShards(DataInputStream is) throws IOException {
        int backwardsCompatibilityBytes = VarInt.readVInt(is);

        if(backwardsCompatibilityBytes == 0)
            return 1;  /// produced by a version of hollow prior to 2.1.0, always only 1 shard.

        skipForwardsCompatibilityBytes(is);

        return VarInt.readVInt(is);
    }

    private void skipForwardsCompatibilityBytes(DataInputStream is) throws IOException {
        int bytesToSkip = VarInt.readVInt(is);
        while(bytesToSkip > 0) {
            int skippedBytes = (int)is.skip(bytesToSkip);
            if(skippedBytes < 0)
                throw new EOFException();
            bytesToSkip -= skippedBytes;
        }
    }

    private void discardDelta(DataInputStream dis, HollowSchema schema, int numShards) throws IOException {
        if(schema instanceof HollowObjectSchema)
            HollowObjectTypeReadState.discardDelta(dis, (HollowObjectSchema)schema, numShards);
        else if(schema instanceof HollowListSchema)
            HollowListTypeReadState.discardDelta(dis, numShards);
        else if(schema instanceof HollowSetSchema)
            HollowSetTypeReadState.discardDelta(dis, numShards);
        else if(schema instanceof HollowMapSchema)
            HollowMapTypeReadState.discardDelta(dis, numShards);
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.diffview;

import com.netflix.hollow.diffview.effigy.HollowEffigy.Field;
import com.netflix.hollow.ui.HollowUISession;
import java.io.IOException;
import java.io.Writer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

public class DiffViewOutputGenerator {

    private final HollowObjectViewProvider viewProvider;

    public DiffViewOutputGenerator(HollowObjectViewProvider viewProvider) {
        this.viewProvider = viewProvider;
    }

    public void collapseRow(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        HollowDiffViewRow row = findRow(req, resp);

        for(HollowDiffViewRow child : row.getChildren())
            child.setVisibility(false);

        resp.getWriter().write("ok");
    }

    public void uncollapseRow(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        HollowDiffViewRow row = findRow(req, resp);

        for(HollowDiffViewRow child : row.getChildren())
            child.setVisibility(true);

        buildChildRowDisplayData(row, resp.getWriter());
    }

    private HollowDiffViewRow findRow(HttpServletRequest req, HttpServletResponse resp) {
        HollowUISession session = HollowUISession.getSession(req, resp);
        HollowObjectView objectView = viewProvider.getObjectView(req, session);

        int rowPath[] = getRowPath(req.getParameter("row"));
        HollowDiffViewRow row = objectView.getRootRow();
        for(int i=0;i<rowPath.length;i++)
            row = row.getChildren().get(rowPath[i]);
        return row;
    }

    private int[] getRowPath(String rowPathStr) {
        String rowPathElementStrings[] = rowPathStr.split("\\.");

        int rowPath[] = new int[rowPathElementStrings.length];

        for(int i=0;i<rowPathElementStrings.length;i++)
            rowPath[i] = Integer.parseInt(rowPathElementStrings[i]);

        return rowPath;
    }

    public static void buildChildRowDisplayData(HollowDiffViewRow parentRow, Writer writer) throws IOException {
        buildChildRowDisplayData(parentRow, writer, true);
    }

    private static void buildChildRowDisplayData(HollowDiffViewRow parentRow, Writer writer, boolean firstRow) throws IOException {

        for(HollowDiffViewRow row : parentRow.getChildren()) {
            if(row.isVisible()) {
                if(firstRow) {
                    firstRow = false;
                } else {
                    writer.write("|");
                }

                writeRowPathString(row, writer);                          writer.write("|");
                writer.write(row.getAvailableAction().toString());        writer.write("|");
                writer.write(marginIdx(row.getFieldPair().getFromIdx())); writer.write("|");
                writer.write(fromCellClassname(row));                     writer.write("|");
                writer.write(fromContent(row));                           writer.write("|");
                writer.write(marginIdx(row.getFieldPair().getToIdx()));   writer.write("|");
                writer.write(toCellClassname(row));                       writer.write("|");
                writer.write(toContent(row));

                buildChildRowDisplayData(row, writer, false);
            }
        }
    }

    private static void writeRowPathString(HollowDiffViewRow row, Writer writer) throws IOException {
        for(int i=0;i<row.getRowPath().length;i++) {
            if(i > 0)
                writer.write('.');
            writer.write(String.valueOf(row.getRowPath()[i]));
        }
    }

    private static String marginIdx(int idx) {
        if(idx == -1)
            return "";
        return String.valueOf(idx);
    }

    private static String fromCellClassname(HollowDiffViewRow currentRow) {
        if(currentRow.getFieldPair().getTo() == null)
            return "delete";
        else if(currentRow.getFieldPair().getFrom() == null)
            return "empty";

        if(currentRow.getFieldPair().getFrom().getValue() == null && currentRow.getFieldPair().getTo().getValue() != null)
            return "replace";
        if(currentRow.getFieldPair().getFrom().getValue() != null && currentRow.getFieldPair().getTo().getValue() == null)
            return "replace";
        if(currentRow.getFieldPair().getFrom().getValue() == null && currentRow.getFieldPair().getTo().getValue() == null)
            return "equal";

        if(currentRow.getFieldPair().isLeafNode() && !currentRow.getFieldPair().getFrom().getValue().equals(currentRow.getFieldPair().getTo().getValue()))
            return "replace";

        return "equal";
    }

    private static String toCellClassname(HollowDiffViewRow currentRow) {
        if(currentRow.getFieldPair().getFrom() == null)
            return "insert";
        else if(currentRow.getFieldPair().getTo() == null)
            return "empty";

        if(currentRow.getFieldPair().getFrom().getValue() == null && currentRow.getFieldPair().getTo().getValue() != null)
            return "replace";
        if(currentRow.getFieldPair().getFrom().getValue() != null && currentRow.getFieldPair().getTo().getValue() == null)
            return "replace";
        if(currentRow.getFieldPair().getFrom().getValue() == null && currentRow.getFieldPair().getTo().getValue() == null)
            return "equal";

        if(currentRow.getFieldPair().isLeafNode() && !currentRow.getFieldPair().getFrom().getValue().equals(currentRow.getFieldPair().getTo().getValue()))
            return "replace";

        return "equal";
    }

    private static String fromContent(HollowDiffViewRow row) {
        boolean moreRows[] = new boolean[row.getIndentation() + 1];
        for(int i=0;i<=row.getIndentation();i++)
            moreRows[i] = row.hasMoreFromRows(i);

        if(row.getFieldPair().getFrom() == null)
            return unpopulatedContent(moreRows);

        String fieldName = row.getFieldPair().getFrom().getFieldName();
        return populatedContent(moreRows, row.getIndentation(),
            row.getFieldPair().isLeafNode(), fieldName, getFieldValue(row, true));
    }

    private static String toContent(HollowDiffViewRow row) {
        boolean moreRows[] = new boolean[row.getIndentation() + 1];
        for(int i=0;i<=row.getIndentation();i++)
            moreRows[i] = row.hasMoreToRows(i);

        if(row.getFieldPair().getTo() == null)
            return unpopulatedContent(moreRows);

        String fieldName = row.getFieldPair().getTo().getFieldName();
        return populatedContent(moreRows, row.getIndentation(), row.getFieldPair().isLeafNode(), fieldName,
            getFieldValue(row, false));
    }

    /**
     * Returns a String representation of the provided row's field value. If `useFrom` is
     * true, this will use the `from` value from the pair, otherwise this will use the
     * `to` value.
     */
    private static String getFieldValue(HollowDiffViewRow row, boolean useFrom) {
        Field field = useFrom ? row.getFieldPair().getFrom() : row.getFieldPair().getTo();
        if (row.getFieldPair().isLeafNode()) {
          return field.getValue() == null ? "null"
              : field.getValue().toString().replace("|", "&#x2502");
        } else {
            String suffix = field.getValue() == null ? " [null]" : "";
            return "(" + field.getTypeName() + ")" + suffix;
        }
    }

    private static String unpopulatedContent(boolean moreRows[]) {
        StringBuilder builder = new StringBuilder();
        for(int i=0;i<moreRows.length;i++) {
            if(moreRows[i]) {
                builder.append(" │");
            } else {
                builder.append("  ");
            }
        }
        return builder.toString();
    }

    private static String populatedContent(boolean moreRows[], int indentation, boolean leafNode, String fieldName, String value) {
        StringBuilder builder = new StringBuilder();
        for(int i=0;i<indentation;i++) {
            if(moreRows[i]) {
                builder.append(".│");
            } else {
                builder.append("..");
            }
        }

        if(!leafNode) {
            if(moreRows[indentation])
                builder.append(".┝━┯━>");
            else
                builder.append(".┕━┯━>");
        } else {
            if(moreRows[indentation])
                builder.append(".├───>");
            else
                builder.append(".└───>");
        }

        if(fieldName != null)
            builder.append(fieldName).append(": ");
        builder.append(value);

        return builder.toString();
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory;

import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import sun.misc.Unsafe;

/**
 * A segmented byte array backs the {@link ByteData} interface with array segments, which potentially come from a pool of reusable memory.

 *
 * This ByteData can grow without allocating successively larger blocks and copying memory.

 *
 * Segment length is always a power of two so that the location of a given index can be found with mask and shift operations.

 *
 * Conceptually this can be thought of as a single byte array of undefined length.  The currently allocated buffer will always be
 * a multiple of the size of the segments.  The buffer will grow automatically when a byte is written to an index greater than the
 * currently allocated buffer.
 *
 * @see ArraySegmentRecycler
 *
 * @author dkoszewnik
 *
 */
@SuppressWarnings("restriction")
public class SegmentedByteArray implements ByteData {

    private static final Unsafe unsafe = HollowUnsafeHandle.getUnsafe();

    private byte[][] segments;
    private final int log2OfSegmentSize;
    private final int bitmask;
    private final ArraySegmentRecycler memoryRecycler;

    public SegmentedByteArray(ArraySegmentRecycler memoryRecycler) {
        this.segments = new byte[2][];
        this.log2OfSegmentSize = memoryRecycler.getLog2OfByteSegmentSize();
        this.bitmask = (1 << log2OfSegmentSize) - 1;
        this.memoryRecycler = memoryRecycler;
    }

    /**
     * Set the byte at the given index to the specified value
     * @param index the index
     * @param value the byte value
     */
    public void set(long index, byte value) {
        int segmentIndex = (int)(index >> log2OfSegmentSize);
        ensureCapacity(segmentIndex);
        segments[segmentIndex][(int)(index & bitmask)] = value;
    }

    /**
     * Get the value of the byte at the specified index.
     * @param index the index
     * @return the byte value
     */
    public byte get(long index) {
        return segments[(int)(index >>> log2OfSegmentSize)][(int)(index & bitmask)];
    }

    /**
     * Copy bytes from another ByteData to this array.
     *
     * @param src the source data
     * @param srcPos the position to begin copying from the source data
     * @param destPos the position to begin writing in this array
     * @param length the length of the data to copy
     */
    public void copy(ByteData src, long srcPos, long destPos, long length) {
        for(long i=0;i<length;i++) {
            set(destPos++, src.get(srcPos++));
        }
    }

    /**
     * For a SegmentedByteArray, this is a faster copy implementation.
     *
     * @param src the source data
     * @param srcPos the position to begin copying from the source data
     * @param destPos the position to begin writing in this array
     * @param length the length of the data to copy
     */
    public void copy(SegmentedByteArray src, long srcPos, long destPos, long length) {
        int segmentLength = 1 << log2OfSegmentSize;
        int currentSegment = (int)(destPos >>> log2OfSegmentSize);
        int segmentStartPos = (int)(destPos & bitmask);
        int remainingBytesInSegment = segmentLength - segmentStartPos;

        while(length > 0) {
            int bytesToCopyFromSegment = (int)Math.min(remainingBytesInSegment, length);
            ensureCapacity(currentSegment);
            int copiedBytes = src.copy(srcPos, segments[currentSegment], segmentStartPos, bytesToCopyFromSegment);

            srcPos += copiedBytes;
            length -= copiedBytes;
            segmentStartPos = 0;
            remainingBytesInSegment = segmentLength;
            currentSegment++;
        }
    }

    /**
     * copies exactly data.length bytes from this SegmentedByteArray into the provided byte array
     *
     * @param srcPos the position to begin copying from the source data
     * @param data the source data
     * @param destPos the position to begin writing in this array
     * @param length the length of the data to copy
     * @return the number of bytes copied
     */
    public int copy(long srcPos, byte[] data, int destPos, int length) {
        int segmentSize = 1 << log2OfSegmentSize;
        int remainingBytesInSegment = (int)(segmentSize - (srcPos & bitmask));
        int dataPosition = destPos;

        while(length > 0) {
            byte[] segment = segments[(int)(srcPos >>> log2OfSegmentSize)];

            int bytesToCopyFromSegment = Math.min(remainingBytesInSegment, length);

            System.arraycopy(segment, (int)(srcPos & bitmask), data, dataPosition, bytesToCopyFromSegment);

            dataPosition += bytesToCopyFromSegment;
            srcPos += bytesToCopyFromSegment;
            remainingBytesInSegment = segmentSize - (int)(srcPos & bitmask);
            length -= bytesToCopyFromSegment;
        }

        return dataPosition - destPos;
    }

    /**
     * checks equality for a specified range of bytes in two arrays
     *
     * @param rangeStart the start position of the comparison range in this array
     * @param compareTo the other array to compare
     * @param cmpStart the start position of the comparison range in the other array
     * @param length the length of the comparison range
     * @return
     */
    public boolean rangeEquals(long rangeStart, SegmentedByteArray compareTo, long cmpStart, int length) {
    	for(int i=0;i<length;i++)
    		if(get(rangeStart + i) != compareTo.get(cmpStart + i))
    			return false;
    	return true;
    }

    /**
     * Copies the data from the provided source array into this array, guaranteeing that
     * if the update is seen by another thread, then all other writes prior to this call
     * are also visible to that thread.
     *
     * @param src the source data
     * @param srcPos the position to begin copying from the source data
     * @param destPos the position to begin writing in this array
     * @param length the length of the data to copy
     */
    public void orderedCopy(SegmentedByteArray src, long srcPos, long destPos, long length) {
        int segmentLength = 1 << log2OfSegmentSize;
        int currentSegment = (int)(destPos >>> log2OfSegmentSize);
        int segmentStartPos = (int)(destPos & bitmask);
        int remainingBytesInSegment = segmentLength - segmentStartPos;

        while(length > 0) {
            int bytesToCopyFromSegment = (int)Math.min(remainingBytesInSegment, length);
            ensureCapacity(currentSegment);
            int copiedBytes = src.orderedCopy(srcPos, segments[currentSegment], segmentStartPos, bytesToCopyFromSegment);

            srcPos += copiedBytes;
            length -= copiedBytes;
            segmentStartPos = 0;
            remainingBytesInSegment = segmentLength;
            currentSegment++;
        }
    }

    /**
     * copies exactly data.length bytes from this SegmentedByteArray into the provided byte array,
     * guaranteeing that if the update is seen by another thread, then all other writes prior to
     * this call are also visible to that thread.
     *
     * @param srcPos the position to begin copying from the source data
     * @param data the source data
     * @param destPos the position to begin writing in this array
     * @param length the length of the data to copy
     * @return the number of bytes copied
     */
    public int orderedCopy(long srcPos, byte[] data, int destPos, int length) {
        int segmentSize = 1 << log2OfSegmentSize;
        int remainingBytesInSegment = (int)(segmentSize - (srcPos & bitmask));
        int dataPosition = destPos;

        while(length > 0) {
            byte[] segment = segments[(int)(srcPos >>> log2OfSegmentSize)];

            int bytesToCopyFromSegment = Math.min(remainingBytesInSegment, length);

            orderedCopy(segment, (int)(srcPos & bitmask), data, dataPosition, bytesToCopyFromSegment);

            dataPosition += bytesToCopyFromSegment;
            srcPos += bytesToCopyFromSegment;
            remainingBytesInSegment = segmentSize - (int)(srcPos & bitmask);
            length -= bytesToCopyFromSegment;
        }

        return dataPosition - destPos;
    }

    /**
     * Copy bytes from the supplied InputStream into this array.
     *
     * @param is the source data
     * @param length the length of the data to copy
     * @throws IOException if the copy could not be performed
     */
    public void readFrom(InputStream is, long length) throws IOException {
        int segmentSize = 1 << log2OfSegmentSize;
        int segment = 0;

        byte scratch[] = new byte[segmentSize];

        while(length > 0) {
            ensureCapacity(segment);
            long bytesToCopy = Math.min(segmentSize, length);
            long bytesCopied = 0;
            while(bytesCopied < bytesToCopy) {
                bytesCopied += is.read(scratch, (int)bytesCopied, (int)(bytesToCopy - bytesCopied));
            }
            orderedCopy(scratch, 0, segments[segment++], 0, (int)bytesCopied);
            length -= bytesCopied;
        }
    }

    /**
     * Write a portion of this data to an OutputStream.
     *
     * @param os the output stream to write to
     * @param startPosition the position to begin copying from this array
     * @param len the length of the data to copy
     * @throws IOException if the write to the output stream could not be performed
     */
    public void writeTo(OutputStream os, long startPosition, long len) throws IOException {
        int segmentSize = 1 << log2OfSegmentSize;
        int remainingBytesInSegment = segmentSize - (int)(startPosition & bitmask);
        long remainingBytesInCopy = len;

        while(remainingBytesInCopy > 0) {
            long bytesToCopyFromSegment = Math.min(remainingBytesInSegment, remainingBytesInCopy);

            os.write(segments[(int)(startPosition >>> log2OfSegmentSize)], (int)(startPosition & bitmask), (int)bytesToCopyFromSegment);

            startPosition += bytesToCopyFromSegment;
            remainingBytesInSegment = segmentSize - (int)(startPosition & bitmask);
            remainingBytesInCopy -= bytesToCopyFromSegment;
        }
    }

    private void orderedCopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) {
        int endSrcPos = srcPos + length;
        destPos += Unsafe.ARRAY_BYTE_BASE_OFFSET;

        while(srcPos < endSrcPos) {
            unsafe.putByteVolatile(dest, destPos++, src[srcPos++]);
        }
    }

    /**
     * Ensures that the segment at segmentIndex exists
     *
     * @param segmentIndex the segment index
     */
    private void ensureCapacity(int segmentIndex) {
        while(segmentIndex >= segments.length) {
            segments = Arrays.copyOf(segments, segments.length * 3 / 2);
        }

        if(segments[segmentIndex] == null) {
            segments[segmentIndex] = memoryRecycler.getByteArray();
        }
    }

    public void destroy() {
        for(int i=0;i<segments.length;i++) {
            if(segments[i] != null)
                memoryRecycler.recycleByteArray(segments[i]);
        }
    }

    public long size() {
        long size = 0;
        for(int i=0;i<segments.length;i++) {
            if(segments[i] != null)
                size += segments[i].length;
        }

        return size;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.schema;

import com.netflix.hollow.api.error.IncompatibleSchemaException;
import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.read.filter.HollowFilterConfig.ObjectFilterConfig;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;

/**
 * A schema for an Object record type.
 *
 * @see HollowSchema
 *
 * @author dkoszewnik
 *
 */
public class HollowObjectSchema extends HollowSchema {

    private final Map<String, Integer> nameFieldIndexLookup;
    private final String fieldNames[];
    private final FieldType fieldTypes[];
    protected final String referencedTypes[];
    private final HollowTypeReadState referencedFieldTypeStates[];  /// populated during deserialization
    private final PrimaryKey primaryKey;

    private int size;

    public HollowObjectSchema(String schemaName, int numFields, String... keyFieldPaths) {
        this(schemaName, numFields, keyFieldPaths == null || keyFieldPaths.length == 0 ? null : new PrimaryKey(schemaName, keyFieldPaths));
    }

    public HollowObjectSchema(String schemaName, int numFields, PrimaryKey primaryKey) {
        super(schemaName);

        this.nameFieldIndexLookup = new HashMap<>(numFields);
        this.fieldNames = new String[numFields];
        this.fieldTypes = new FieldType[numFields];
        this.referencedTypes = new String[numFields];
        this.referencedFieldTypeStates = new HollowTypeReadState[numFields];
        this.primaryKey = primaryKey;
    }

    public int numFields() {
        return size;
    }

    public PrimaryKey getPrimaryKey() {
        return primaryKey;
    }

    public int addField(String fieldName, FieldType fieldType) {
        return addField(fieldName, fieldType, null);
    }

    public int addField(String fieldName, FieldType fieldType, String referencedType) {
        if (fieldType == FieldType.REFERENCE && referencedType == null) {
            throw new RuntimeException(
                    "When adding a REFERENCE field to a schema, the referenced type must be provided.  Check type: "
                            + getName() + " field: " + fieldName);
        }
        fieldNames[size] = fieldName;
        fieldTypes[size] = fieldType;
        referencedTypes[size] = referencedType;
        nameFieldIndexLookup.put(fieldName, size);

        size++;

        return size - 1;
    }

    /**
     * @deprecated This method ignores the provided {@code HollowTypeReadState} - you should use
     * {@link #addField(String, FieldType, String)} and then call
     * {@link #setReferencedTypeState(int, HollowTypeReadState)} with the returned integer. This
     * method will be removed in a future release.
     */
    @Deprecated
    public int addField(String fieldName, FieldType fieldType, String referencedType, HollowTypeReadState referencedTypeState) {
        return addField(fieldName, fieldType, referencedType);
    }

    /**
     * Returns the position of a field previously added to the map, or -1 if the field has not been added to the map.
     *
     * The positions of the fields are hashed into the hashedPositionArray by the hashCode of the fieldName.
     *
     * @param fieldName the field name
     * @return the position
     */
    public int getPosition(String fieldName) {
        Integer index = nameFieldIndexLookup.get(fieldName);
        if (index == null) {
            return -1;
        }
        return index;
    }

    public String getFieldName(int fieldPosition) {
        return fieldNames[fieldPosition];
    }

    public FieldType getFieldType(String fieldName) {
        int fieldPosition = getPosition(fieldName);

        if(fieldPosition == -1)
            return null;

        return getFieldType(fieldPosition);
    }

    public FieldType getFieldType(int fieldPosition) {
        return fieldTypes[fieldPosition];
    }

    public String getReferencedType(String fieldName) {
        int fieldPosition = getPosition(fieldName);

        if(fieldPosition == -1)
            return null;

        return getReferencedType(fieldPosition);
    }

    public String getReferencedType(int fieldPosition) {
        return referencedTypes[fieldPosition];
    }

    public void setReferencedTypeState(int fieldPosition, HollowTypeReadState state) {
        referencedFieldTypeStates[fieldPosition] = state;
    }

    public HollowTypeReadState getReferencedTypeState(int fieldPosition) {
        return referencedFieldTypeStates[fieldPosition];
    }

    public HollowObjectSchema findCommonSchema(HollowObjectSchema otherSchema) {
        if(!getName().equals(otherSchema.getName())) {
            throw new IllegalArgumentException("Cannot find common schema of two schemas with different names!");
        }

        int commonFields = 0;

        for (String fieldName : fieldNames) {
            if(otherSchema.getPosition(fieldName) != -1) {
                commonFields++;
            }
        }

        PrimaryKey primaryKey = isNullableObjectEquals(this.primaryKey, otherSchema.getPrimaryKey()) ? this.primaryKey : null;
        HollowObjectSchema commonSchema = new HollowObjectSchema(getName(), commonFields, primaryKey);

        for (int i = 0; i < fieldNames.length; i++) {
            int otherFieldIndex = otherSchema.getPosition(fieldNames[i]);
             if (otherFieldIndex != -1) {
                if (fieldTypes[i] != otherSchema.getFieldType(otherFieldIndex)
                        || !referencedTypesEqual(referencedTypes[i], otherSchema.getReferencedType(otherFieldIndex))) {
                    String fieldType = fieldTypes[i] == FieldType.REFERENCE ? referencedTypes[i]
                        : fieldTypes[i].toString().toLowerCase();
                    String otherFieldType = otherSchema.getFieldType(otherFieldIndex) == FieldType.REFERENCE
                        ? otherSchema.getReferencedType(otherFieldIndex)
                        : otherSchema.getFieldType(otherFieldIndex).toString().toLowerCase();
                    throw new IncompatibleSchemaException(getName(), fieldNames[i], fieldType, otherFieldType);
                }

                commonSchema.addField(fieldNames[i], fieldTypes[i], referencedTypes[i]);
            }
        }

        return commonSchema;
    }

    public HollowObjectSchema findUnionSchema(HollowObjectSchema otherSchema) {
        if(!getName().equals(otherSchema.getName())) {
            throw new IllegalArgumentException("Cannot find common schema of two schemas with different names!");
        }

        int totalFields = otherSchema.numFields();

        for (String fieldName : fieldNames) {
            if(otherSchema.getPosition(fieldName) == -1)
                totalFields++;
        }

        PrimaryKey primaryKey = isNullableObjectEquals(this.primaryKey, otherSchema.getPrimaryKey()) ? this.primaryKey : null;
        HollowObjectSchema unionSchema = new HollowObjectSchema(getName(), totalFields, primaryKey);

        for(int i=0;i<fieldNames.length;i++) {
            unionSchema.addField(fieldNames[i], fieldTypes[i], referencedTypes[i]);
        }

        for(int i=0;i<otherSchema.numFields();i++) {
            if(getPosition(otherSchema.getFieldName(i)) == -1) {
                unionSchema.addField(otherSchema.getFieldName(i), otherSchema.getFieldType(i), otherSchema.getReferencedType(i));
            }
        }

        return unionSchema;
    }

    public HollowObjectSchema filterSchema(HollowFilterConfig config) {
        ObjectFilterConfig typeConfig = config.getObjectTypeConfig(getName());

        int includedFields = 0;

        for(int i=0;i<numFields();i++) {
            if(typeConfig.includesField(getFieldName(i)))
                includedFields++;
        }

        HollowObjectSchema filteredSchema = new HollowObjectSchema(getName(), includedFields, primaryKey);

        for(int i=0;i<numFields();i++) {
            if(typeConfig.includesField(getFieldName(i)))
                filteredSchema.addField(getFieldName(i), getFieldType(i), getReferencedType(i));
        }

        return filteredSchema;
    }

    private boolean referencedTypesEqual(String type1, String type2) {
        if(type1 == null)
            return type2 == null;
        return type1.equals(type2);
    }

    @Override
    public SchemaType getSchemaType() {
        return SchemaType.OBJECT;
    }

    @Override
    public boolean equals(Object other) {
        if (this == other)
            return true;
        if(!(other instanceof HollowObjectSchema))
            return false;
        HollowObjectSchema otherSchema = (HollowObjectSchema) other;
        if(!getName().equals(otherSchema.getName()))
            return false;
        if(otherSchema.numFields() != numFields())
            return false;

        if (!isNullableObjectEquals(primaryKey, otherSchema.getPrimaryKey()))
            return false;

        for(int i=0;i<numFields();i++) {
            if(getFieldType(i) != otherSchema.getFieldType(i))
                return false;
            if(!getFieldName(i).equals(otherSchema.getFieldName(i)))
                return false;
        }

        return true;
    }

    @Override
    public String toString() {
        StringBuilder builder = new StringBuilder();

        builder.append(getName());
        if (primaryKey != null) {
            builder.append(" @PrimaryKey(");
            if (primaryKey.numFields() > 0) {
                builder.append(primaryKey.getFieldPath(0));
                for (int i = 1; i < primaryKey.numFields(); i++) {
                    builder.append(", ").append(primaryKey.getFieldPath(i));
                }
            }
            builder.append(")");
        }

        builder.append(" {\n");
        for(int i=0;i<numFields();i++) {
            builder.append("\t");
            if(getFieldType(i) == FieldType.REFERENCE) {
                builder.append(getReferencedType(i));
            } else {
                builder.append(getFieldType(i).toString().toLowerCase());
            }
            builder.append(" ").append(getFieldName(i)).append(";\n");
        }
        builder.append("}");

        return builder.toString();
    }

    @Override
    public void writeTo(OutputStream os) throws IOException {
        DataOutputStream dos = new DataOutputStream(os);

        if (primaryKey != null)
            dos.write(SchemaType.OBJECT.getTypeIdWithPrimaryKey());
        else
            dos.write(SchemaType.OBJECT.getTypeId());

        dos.writeUTF(getName());
        if (primaryKey != null) {
            VarInt.writeVInt(dos, primaryKey.numFields());
            for (int i = 0; i < primaryKey.numFields(); i++) {
                dos.writeUTF(primaryKey.getFieldPath(i));
            }
        }

        dos.writeShort(size);
        for(int i=0;i<size;i++) {
            dos.writeUTF(fieldNames[i]);
            dos.writeUTF(fieldTypes[i].name());
            if(fieldTypes[i] == FieldType.REFERENCE)
                dos.writeUTF(referencedTypes[i]);
        }

    }

    /**
     * All allowable field types.
     *
     */
    public enum FieldType {
        /**
         * A reference to another field.  References are typed, and are fixed-length fields are encoded as the ordinal of the referenced record.
         */
        REFERENCE(-1, false),
        /**
         * An integer value up to 32 bits.  Integers are fixed-length fields encoded with zig-zag encoding.
         * The value Integer.MIN_VALUE is reserved for a sentinel value indicating null.
         */
        INT(-1, false),
        /**
         * An integer value up to 64 bits.  Longs are fixed-length fields encoded with zig-zag encoding.
         * The value Long.MIN_VALUE is reserved for a sentinel value indicating null.
         */
        LONG(-1, false),
        /**
         * A boolean value.  Booleans are encoded as fields requiring two bits each.  Two bits are required
         * because boolean fields can carry any of the three values: true, false, or null.
         */
        BOOLEAN(1, false),
        /**
         * A floating-point number.  Floats are encoded as fixed-length fields four bytes long.
         */
        FLOAT(4, false),
        /**
         * A double-precision floating point number.  Doubles are encoded as fixed-length fields eight bytes long.
         */
        DOUBLE(8, false),
        /**
         * A String of characters.  All Strings for all records containing a given field are encoded in a packed array
         * of variable-length characters.  The values are ordered by the ordinal of the record to which they belong.
         * Each individual record contains a fixed-length field which holds an integer which points to the end of the
         * array range containing the value for the specific record.  The beginning of the range is determined by
         * reading the pointer from the previous record.
         */
        STRING(-1, true),
        /**
         * A byte array.  All byte arrays for all records containing a given field are encoded in a packed array
         * of bytes.  The values are ordered by the ordinal of the record to which they belong.
         * Each individual record contains a fixed-length field which holds an integer which points to the end of the
         * array range containing the value for the specific record.  The beginning of the range is determined by
         * reading the pointer from the previous record.
         */
        BYTES(-1, true);

        private final int fixedLength;
        private final boolean varIntEncodesLength;

        FieldType(int fixedLength, boolean varIntEncodesLength) {
            this.fixedLength = fixedLength;
            this.varIntEncodesLength = varIntEncodesLength;
        }

        public int getFixedLength() {
            return fixedLength;
        }

        public boolean isVariableLength() {
            return varIntEncodesLength;
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import static com.netflix.hollow.core.memory.encoding.FixedLengthElementArray.bitsRequiredToRepresentValue;

import com.netflix.hollow.core.index.traversal.HollowIndexerValueTraverser;
import com.netflix.hollow.core.memory.encoding.FixedLengthElementArray;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;
import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.dataaccess.HollowObjectTypeDataAccess;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
import java.util.BitSet;

public class HollowHashIndexBuilder {

    private final HollowPreindexer preindexer;

    private final int[] bitsPerTraverserField;
    private final int[] offsetPerTraverserField;
    private final int bitsPerMatchHashKey;
    final int bitsPerSelectHashEntry;

    private final ArraySegmentRecycler memoryRecycler;

    private FixedLengthElementArray finalMatchHashTable;
    FixedLengthElementArray finalSelectHashArray;
    private long finalMatchHashMask;
    private int finalBitsPerMatchHashEntry;
    private int finalBitsPerSelectTableSize;
    private int finalBitsPerSelectTablePointer;

    private GrowingSegmentedLongArray matchIndexHashAndSizeArray;
    private FixedLengthElementArray intermediateMatchHashTable;
    private MultiLinkedElementArray intermediateSelectLists;
    private int intermediateMatchHashTableSize;
    private int bitsPerIntermediateListIdentifier;
    private int bitsPerIntermediateMatchHashEntry;
    private int intermediateMatchHashMask;
    private int intermediateMatchHashTableSizeBeforeGrow;
    private int matchCount;

    ///TODO: Optimization, make the matchFields[].schemaFieldPositionPath as short as possible, to reduce iteration
    /// this means merging the common roots of path from the same base field, and pushing all unique base fields down
    /// to the leaves.
    public HollowHashIndexBuilder(HollowReadStateEngine stateEngine, String type, String selectField, String... matchFields) {
        this.preindexer = new HollowPreindexer(stateEngine, type, selectField, matchFields);
        preindexer.buildFieldSpecifications();

        this.memoryRecycler = WastefulRecycler.DEFAULT_INSTANCE;

        HollowIndexerValueTraverser traverser = preindexer.getTraverser();

        this.bitsPerTraverserField = new int[traverser.getNumFieldPaths()];
        this.offsetPerTraverserField = new int[traverser.getNumFieldPaths()];

        int bitsPerMatchHashKey = 0;
        for(int i=0;i<traverser.getNumFieldPaths();i++) {
            int maxOrdinalForTypeState = ((HollowTypeReadState)traverser.getFieldTypeDataAccess(i)).maxOrdinal();
            bitsPerTraverserField[i] = bitsRequiredToRepresentValue(maxOrdinalForTypeState + 1);
            offsetPerTraverserField[i] = bitsPerMatchHashKey;
            if(i < preindexer.getNumMatchTraverserFields())
                bitsPerMatchHashKey += bitsPerTraverserField[i];
        }

        this.bitsPerMatchHashKey = bitsPerMatchHashKey;
        this.bitsPerSelectHashEntry = bitsPerTraverserField[preindexer.getSelectFieldSpec().getBaseIteratorFieldIdx()];
    }

    public void buildIndex() {
        matchIndexHashAndSizeArray = new GrowingSegmentedLongArray(memoryRecycler);

        BitSet populatedOrdinals = preindexer.getTypeState().getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();

        /// an initial guess at how big this table might be -- one match per top-level element.
        int guessNumberOfMatches = populatedOrdinals.cardinality();
        intermediateMatchHashTableSize = HashCodes.hashTableSize(guessNumberOfMatches);
        bitsPerIntermediateListIdentifier =  bitsRequiredToRepresentValue(intermediateMatchHashTableSize - 1);
        bitsPerIntermediateMatchHashEntry = bitsPerMatchHashKey + bitsPerIntermediateListIdentifier;

        intermediateMatchHashMask = intermediateMatchHashTableSize - 1;
        intermediateMatchHashTableSizeBeforeGrow = intermediateMatchHashTableSize * 7 / 10;
        matchCount = 0;

        /// a data structure which keeps canonical matches for comparison (the matchHashTable)
        intermediateMatchHashTable = new FixedLengthElementArray(memoryRecycler, (long)intermediateMatchHashTableSize * bitsPerIntermediateMatchHashEntry);

        /// a data structure which tracks lists of matches under canonical matches.
        intermediateSelectLists = new MultiLinkedElementArray(memoryRecycler);

        HollowIndexerValueTraverser traverser = preindexer.getTraverser();

        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            traverser.traverse(ordinal);

            for(int i=0;i<traverser.getNumMatches();i++) {
                int matchHash = getMatchHash(i);

                long bucket = matchHash & intermediateMatchHashMask;
                long hashBucketBit = bucket * bitsPerIntermediateMatchHashEntry;
                boolean bucketIsEmpty = intermediateMatchHashTable.getElementValue(hashBucketBit, bitsPerTraverserField[0]) == 0;
                long bucketMatchListIdx = intermediateMatchHashTable.getElementValue(hashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier);
                int bucketMatchHashCode = (int)matchIndexHashAndSizeArray.get(bucketMatchListIdx);

                while(!bucketIsEmpty && (bucketMatchHashCode != (matchHash & Integer.MAX_VALUE) || !intermediateMatchIsEqual(i, hashBucketBit))) {
                    bucket = (bucket + 1) & intermediateMatchHashMask;
                    hashBucketBit = bucket * bitsPerIntermediateMatchHashEntry;
                    bucketIsEmpty = intermediateMatchHashTable.getElementValue(hashBucketBit, bitsPerTraverserField[0]) == 0;
                    bucketMatchListIdx = intermediateMatchHashTable.getElementValue(hashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier);
                    bucketMatchHashCode = (int)matchIndexHashAndSizeArray.get(bucketMatchListIdx);
                }

                int matchListIdx;

                if(bucketIsEmpty) {
                    matchListIdx = intermediateSelectLists.newList();
                    for(int j=0;j<preindexer.getNumMatchTraverserFields();j++)
                        intermediateMatchHashTable.setElementValue(hashBucketBit + offsetPerTraverserField[j], bitsPerTraverserField[j], traverser.getMatchOrdinal(i, j) + 1);

                    intermediateMatchHashTable.setElementValue(hashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier, matchListIdx);

                    matchIndexHashAndSizeArray.set(matchListIdx, matchHash & Integer.MAX_VALUE);
                    matchCount++;

                    /// GROW IF NECESSARY!
                    if(matchCount > intermediateMatchHashTableSizeBeforeGrow) {
                        growIntermediateHashTable();
                    }

                } else {
                    matchListIdx = (int)intermediateMatchHashTable.getElementValue(hashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier);
                }

                intermediateSelectLists.add(matchListIdx, traverser.getMatchOrdinal(i, preindexer.getSelectFieldSpec().getBaseIteratorFieldIdx()));
            }

            ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
        }

        /// turn those data structures into a compact one optimized for hash lookup
        long totalNumberOfSelectBucketsAndBitsRequiredForSelectTableSize = calculateDedupedSizesAndTotalNumberOfSelectBuckets(intermediateSelectLists, matchIndexHashAndSizeArray);
        long totalNumberOfSelectBuckets = totalNumberOfSelectBucketsAndBitsRequiredForSelectTableSize & 0xFFFFFFFFFFFFFFL;
        long totalNumberOfMatchBuckets = HashCodes.hashTableSize(matchCount);

        int bitsPerFinalSelectBucketPointer = bitsRequiredToRepresentValue(totalNumberOfSelectBuckets);
        int bitsPerSelectTableSize = (int)(totalNumberOfSelectBucketsAndBitsRequiredForSelectTableSize >>> 56);
        int finalBitsPerMatchHashEntry = bitsPerMatchHashKey + bitsPerSelectTableSize + bitsPerFinalSelectBucketPointer;

        FixedLengthElementArray finalMatchArray = new FixedLengthElementArray(memoryRecycler, totalNumberOfMatchBuckets * finalBitsPerMatchHashEntry);
        FixedLengthElementArray finalSelectArray = new FixedLengthElementArray(memoryRecycler, totalNumberOfSelectBuckets * bitsPerSelectHashEntry);

        long finalMatchHashMask = totalNumberOfMatchBuckets - 1;

        long currentSelectArrayBucket = 0;

        for(int i=0;i<matchCount;i++) {
            long matchIndexHashAndSize = matchIndexHashAndSizeArray.get(i);
            int matchIndexSize = (int)(matchIndexHashAndSize >> 32);
            int matchIndexTableSize = HashCodes.hashTableSize(matchIndexSize);
            int matchIndexBucketMask = matchIndexTableSize - 1;

            HollowOrdinalIterator selectOrdinalIter = intermediateSelectLists.iterator(i);
            int selectOrdinal = selectOrdinalIter.next();
            while(selectOrdinal != HollowOrdinalIterator.NO_MORE_ORDINALS) {
                int selectBucket = HashCodes.hashInt(selectOrdinal) & matchIndexBucketMask;
                int bucketOrdinal = (int)finalSelectArray.getElementValue((currentSelectArrayBucket + selectBucket) * bitsPerSelectHashEntry, bitsPerSelectHashEntry) - 1;
                while(bucketOrdinal != -1 && bucketOrdinal != selectOrdinal) {
                    ///TODO: If select field type is not REFERENCE, then we should dedup -- unless we are reference counting for delta application
                    ///ordinals here with the same value for the specified field.
                    selectBucket = (selectBucket + 1) & matchIndexBucketMask;
                    bucketOrdinal = (int)finalSelectArray.getElementValue((currentSelectArrayBucket + selectBucket) * bitsPerSelectHashEntry, bitsPerSelectHashEntry) - 1;
                }

                if(bucketOrdinal == -1)
                    finalSelectArray.setElementValue((currentSelectArrayBucket + selectBucket) * bitsPerSelectHashEntry, bitsPerSelectHashEntry, selectOrdinal + 1);

                selectOrdinal = selectOrdinalIter.next();
            }

            long finalMatchIndexBucket = matchIndexHashAndSize & finalMatchHashMask;
            long finalMatchIndexBucketBit = finalMatchIndexBucket * finalBitsPerMatchHashEntry;

            while(finalMatchArray.getElementValue(finalMatchIndexBucketBit, bitsPerTraverserField[0]) != 0) {
                finalMatchIndexBucket = (finalMatchIndexBucket + 1) & finalMatchHashMask;
                finalMatchIndexBucketBit = finalMatchIndexBucket * finalBitsPerMatchHashEntry;
            }

            long intermediateMatchHashBucket = matchIndexHashAndSize & intermediateMatchHashMask;
            long intermediateMatchIndexBucketBit = intermediateMatchHashBucket * bitsPerIntermediateMatchHashEntry;
            while(intermediateMatchHashTable.getElementValue(intermediateMatchIndexBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier) != i) {
                intermediateMatchHashBucket = (intermediateMatchHashBucket + 1) & intermediateMatchHashMask;
                intermediateMatchIndexBucketBit = intermediateMatchHashBucket * bitsPerIntermediateMatchHashEntry;
            }

            if(bitsPerMatchHashKey < 56) {
                long matchHashKey = intermediateMatchHashTable.getElementValue(intermediateMatchIndexBucketBit, bitsPerMatchHashKey);
                finalMatchArray.setElementValue(finalMatchIndexBucketBit, bitsPerMatchHashKey, matchHashKey);
            } else {
                finalMatchArray.copyBits(intermediateMatchHashTable, intermediateMatchIndexBucketBit, finalMatchIndexBucketBit, bitsPerMatchHashKey);
            }

            finalMatchArray.setElementValue(finalMatchIndexBucketBit + bitsPerMatchHashKey, bitsPerSelectTableSize, matchIndexSize);
            finalMatchArray.setElementValue(finalMatchIndexBucketBit + bitsPerMatchHashKey + bitsPerSelectTableSize, bitsPerFinalSelectBucketPointer, currentSelectArrayBucket);

            currentSelectArrayBucket += matchIndexTableSize;
        }

        this.finalMatchHashTable = finalMatchArray;
        this.finalSelectHashArray = finalSelectArray;
        this.finalBitsPerMatchHashEntry = finalBitsPerMatchHashEntry;
        this.finalBitsPerSelectTablePointer = bitsPerFinalSelectBucketPointer;
        this.finalBitsPerSelectTableSize = bitsPerSelectTableSize;
        this.finalMatchHashMask = finalMatchHashMask;
    }

    private void growIntermediateHashTable() {
        int newMatchHashTableSize = intermediateMatchHashTableSize * 2;
        int newMatchHashMask = newMatchHashTableSize - 1;
        int newBitsForListIdentifier = bitsRequiredToRepresentValue(newMatchHashTableSize - 1);
        int newBitsPerMatchHashEntry = bitsPerMatchHashKey + newBitsForListIdentifier;
        FixedLengthElementArray newMatchHashTable = new FixedLengthElementArray(memoryRecycler, (long)newMatchHashTableSize * newBitsPerMatchHashEntry);

        for(int j=0;j<matchCount;j++) {
            int rehashCode = (int)matchIndexHashAndSizeArray.get(j);
            long oldHashBucket = rehashCode & intermediateMatchHashMask;
            long oldHashBucketBit = oldHashBucket * bitsPerIntermediateMatchHashEntry;

            while(intermediateMatchHashTable.getElementValue(oldHashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier) != j) {
                oldHashBucket = (oldHashBucket+1) & intermediateMatchHashMask;
                oldHashBucketBit = oldHashBucket * bitsPerIntermediateMatchHashEntry;
            }

            long rehashBucket = rehashCode & newMatchHashMask;
            long rehashBucketBit = rehashBucket * newBitsPerMatchHashEntry;
            boolean rehashBucketIsEmpty = newMatchHashTable.getElementValue(rehashBucketBit, bitsPerTraverserField[0]) == 0;

            while(!rehashBucketIsEmpty) {
                rehashBucket = (rehashBucket + 1) & newMatchHashMask;
                rehashBucketBit = rehashBucket * newBitsPerMatchHashEntry;
                rehashBucketIsEmpty = newMatchHashTable.getElementValue(rehashBucketBit, bitsPerTraverserField[0]) == 0;
            }

            if(bitsPerMatchHashKey < 56) {
                newMatchHashTable.setElementValue(rehashBucketBit, bitsPerMatchHashKey, intermediateMatchHashTable.getElementValue(oldHashBucketBit, bitsPerMatchHashKey));
            } else {
                newMatchHashTable.copyBits(intermediateMatchHashTable, oldHashBucketBit, rehashBucketBit, bitsPerMatchHashKey);
            }

            int listIdx = (int)intermediateMatchHashTable.getElementValue(oldHashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier);
            newMatchHashTable.setElementValue(rehashBucketBit + bitsPerMatchHashKey, bitsPerIntermediateListIdentifier, listIdx);
        }

        intermediateMatchHashTable.destroy(memoryRecycler);
        memoryRecycler.swap();

        intermediateMatchHashTable = newMatchHashTable;
        intermediateMatchHashTableSize = newMatchHashTableSize;
        intermediateMatchHashTableSizeBeforeGrow = intermediateMatchHashTableSize * 7 / 10;
        bitsPerIntermediateListIdentifier = newBitsForListIdentifier;
        bitsPerIntermediateMatchHashEntry = newBitsPerMatchHashEntry;
        intermediateMatchHashMask = newMatchHashMask;
    }

    /**
     * Called after initial pass.
     * Returns the sum total number of select buckets in the low 7 bytes, and the bits required for the max set size in the high 1 byte.
     */
    private long calculateDedupedSizesAndTotalNumberOfSelectBuckets(MultiLinkedElementArray elementArray, GrowingSegmentedLongArray matchIndexHashAndSizeArray) {
        long totalBuckets = 0;
        long maxSize = 0;
        int[] selectArray = new int[8];

        for(int i=0;i<elementArray.numLists();i++) {
            int listSize = elementArray.listSize(i);
            int setSize = 0;
            int predictedBuckets = HashCodes.hashTableSize(listSize);
            int hashMask = predictedBuckets - 1;
            if(predictedBuckets > selectArray.length)
                selectArray = new int[predictedBuckets];
            for(int j=0;j<predictedBuckets;j++)
                selectArray[j] = -1;

            HollowOrdinalIterator iter = elementArray.iterator(i);
            int selectOrdinal = iter.next();
            while(selectOrdinal != HollowOrdinalIterator.NO_MORE_ORDINALS) {
                int hash = HashCodes.hashInt(selectOrdinal);
                int bucket = hash & hashMask;

                while(true) {
                    if(selectArray[bucket] == selectOrdinal)
                        break;
                    if(selectArray[bucket] == -1) {
                        selectArray[bucket] = selectOrdinal;
                        setSize++;
                        break;
                    }

                    bucket = (bucket+1) & hashMask;
                }

                selectOrdinal = iter.next();
            }

            long matchIndexHashAndSize = matchIndexHashAndSizeArray.get(i);
            matchIndexHashAndSize |= (long)setSize << 32;
            matchIndexHashAndSizeArray.set(i, matchIndexHashAndSize);

            totalBuckets += HashCodes.hashTableSize(setSize);
            if(setSize > maxSize)
                maxSize = setSize;
        }

        return totalBuckets | (long)bitsRequiredToRepresentValue(maxSize) << 56;
    }

    private boolean intermediateMatchIsEqual(int matchIdx, long hashBucketBit) {
        for(int i=0;i<preindexer.getMatchFieldSpecs().length;i++) {
            HollowHashIndexField field = preindexer.getMatchFieldSpecs()[i];
            int matchOrdinal = preindexer.getTraverser().getMatchOrdinal(matchIdx, field.getBaseIteratorFieldIdx());
            int hashOrdinal = (int)intermediateMatchHashTable.getElementValue(hashBucketBit + offsetPerTraverserField[field.getBaseIteratorFieldIdx()], bitsPerTraverserField[field.getBaseIteratorFieldIdx()]) - 1;

            HollowTypeReadState readState = field.getBaseDataAccess();
            int[] fieldPath = field.getSchemaFieldPositionPath();

            if(fieldPath.length == 0) {
                if(matchOrdinal != hashOrdinal)
                    return false;
            } else {
                for(int j=0;j<fieldPath.length - 1;j++) {
                    HollowObjectTypeReadState objectAccess = (HollowObjectTypeReadState)readState;
                    readState = objectAccess.getSchema().getReferencedTypeState(fieldPath[j]);
                    matchOrdinal = objectAccess.readOrdinal(matchOrdinal, fieldPath[j]);
                    hashOrdinal = objectAccess.readOrdinal(hashOrdinal, fieldPath[j]);
                }

                if(matchOrdinal != hashOrdinal) {
                    HollowObjectTypeReadState objectAccess = (HollowObjectTypeReadState)readState;
                    int fieldIdx = fieldPath[fieldPath.length-1];
                    if(isAnyFieldNull(matchOrdinal, hashOrdinal) || !HollowReadFieldUtils.fieldsAreEqual(objectAccess, matchOrdinal, fieldIdx, objectAccess, hashOrdinal, fieldIdx))
                        return false;
                }
            }
        }

        return true;
    }

    private boolean isAnyFieldNull(int matchOrdinal, int hashOrdinal) {
        return matchOrdinal == -1 || hashOrdinal == -1;
    }

    private int getMatchHash(int matchIdx) {
        int matchHash = 0;

        for(int i=0;i<preindexer.getMatchFieldSpecs().length;i++) {
            HollowHashIndexField field = preindexer.getMatchFieldSpecs()[i];
            int ordinal = preindexer.getTraverser().getMatchOrdinal(matchIdx, field.getBaseIteratorFieldIdx());
            HollowTypeReadState readState = field.getBaseDataAccess();
            int[] fieldPath = field.getSchemaFieldPositionPath();

            if(fieldPath.length == 0) {
                matchHash ^= HashCodes.hashInt(ordinal);
            } else {
                for(int j=0;j<fieldPath.length-1;j++) {
                    HollowObjectTypeReadState objectAccess = (HollowObjectTypeReadState)readState;
                    readState = objectAccess.getSchema().getReferencedTypeState(fieldPath[j]);
                    ordinal = objectAccess.readOrdinal(ordinal, fieldPath[j]);
                }

                int fieldHashCode = ordinal == -1 ? -1 : HollowReadFieldUtils.fieldHashCode((HollowObjectTypeDataAccess) readState, ordinal, fieldPath[fieldPath.length-1]);
                matchHash ^= HashCodes.hashInt(fieldHashCode);
            }
        }

        return matchHash;
    }

    public int getBitsPerMatchHashKey() {
        return bitsPerMatchHashKey;
    }

    public FixedLengthElementArray getFinalMatchHashTable() {
        return finalMatchHashTable;
    }

    public long getFinalMatchHashMask() {
        return finalMatchHashMask;
    }

    public int getFinalBitsPerMatchHashEntry() {
        return finalBitsPerMatchHashEntry;
    }

    public int getFinalBitsPerSelectTableSize() {
        return finalBitsPerSelectTableSize;
    }

    public int getFinalBitsPerSelectTablePointer() {
        return finalBitsPerSelectTablePointer;
    }

    public FixedLengthElementArray getFinalSelectHashArray() {
        return finalSelectHashArray;
    }

    public HollowHashIndexField getSelectField() {
        return preindexer.getSelectFieldSpec();
    }

    public HollowHashIndexField[] getMatchFields() {
        return preindexer.getMatchFieldSpecs();
    }

    public int[] getBitsPerTraverserField() {
        return bitsPerTraverserField;
    }

    public int[] getOffsetPerTraverserField() {
        return offsetPerTraverserField;
    }

    public int getBitsPerSelectHashEntry() {
        return bitsPerSelectHashEntry;
    }
}