/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory;

import java.util.Arrays;

/**
 * A stack of unused ordinals.<p>
 *
 * This data structure is used by the {@link ByteArrayOrdinalMap} to track and assign unused ordinals to new records.
 *
 * The goal is to ensure the "holes" generated by removing unused ordinals during server processing are reused in subsequent cycles,
 * instead of growing the "ordinal space" indefinitely.
 *
 * @author dkoszewnik
 *
 */
public class FreeOrdinalTracker {

    private int freeOrdinals[];
    private int size;
    private int nextEmptyOrdinal;

    public FreeOrdinalTracker() {
        this(0);
    }

    private FreeOrdinalTracker(int nextEmptyOrdinal) {
        this.freeOrdinals = new int[64];
        this.nextEmptyOrdinal = nextEmptyOrdinal;
        this.size = 0;
    }

    /**
     * @return either an ordinal which was previously deallocated, or the next empty, previously unallocated ordinal in the sequence 0-n
     */
    public int getFreeOrdinal() {
        if(size == 0)
            return nextEmptyOrdinal++;

        return freeOrdinals[--size];
    }

    /**
     * Return an ordinal to the pool after the object to which it was assigned is discarded.
     *
     * @param ordinal the ordinal
     */
    public void returnOrdinalToPool(int ordinal) {
        if(size == freeOrdinals.length) {
            freeOrdinals = Arrays.copyOf(freeOrdinals, freeOrdinals.length * 3 / 2);
        }

        freeOrdinals[size] = ordinal;
        size++;
    }

    /**
     * Specify the next ordinal to return after the reusable pool is exhausted
     *
     * @param nextEmptyOrdinal the next empty ordinal
     */
    public void setNextEmptyOrdinal(int nextEmptyOrdinal) {
        this.nextEmptyOrdinal = nextEmptyOrdinal;
    }

    /**
     * Ensure that all future ordinals are returned in ascending order.
     */
    public void sort() {
        Arrays.sort(freeOrdinals, 0, size);

        /// reverse the ordering
        int midpoint = size / 2;
        for(int i=0;i<midpoint;i++) {
            int temp = freeOrdinals[i];
            freeOrdinals[i] = freeOrdinals[size-i-1];
            freeOrdinals[size-i-1] = temp;
        }
    }

    /**
     * Resets the FreeOrdinalTracker to its initial state.
     */
    public void reset() {
        size = 0;
        nextEmptyOrdinal = 0;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;

/**
 * Hashes POJO keys according to the Set/Map hash key specification.  This is used
 * in order to lookup matching records in these hashtable data structures.
 */
public class SetMapKeyHasher {

    /**
     * Hash a key
     *
     * @param key the key
     * @param fieldType the respective FieldTypes of each key.
     * @return the hash code
     */
    public static int hash(Object key[], FieldType fieldType[]) {
        int hash = 0;

        for (int i = 0; i < key.length; i++) {
            hash *= 31;
            hash ^= hash(key[i], fieldType[i]);
        }

        return hash;
    }

    /**
     * Hash a single key field
     *
     * @param key the key
     * @param fieldType the FieldType of the key.
     * @return the hash code
     */
    public static int hash(Object key, FieldType fieldType) {
        switch(fieldType) {
        case INT:
            return HashCodes.hashInt(((Integer)key).intValue());
        case LONG:
            long longVal = ((Long)key).longValue();
            return HashCodes.hashInt((int)(longVal ^ (longVal >>> 32)));
        case REFERENCE:
            return HashCodes.hashInt(((Integer)key).intValue());
        case BYTES:
            return HashCodes.hashInt(HashCodes.hashCode((byte[])key));
        case STRING:
            return HashCodes.hashInt(key.hashCode());
        case BOOLEAN:
            return HashCodes.hashInt(((Boolean)key).booleanValue() ? 1231 : 1237);
        case DOUBLE:
            long longBits = Double.doubleToRawLongBits(((Double)key).doubleValue());
            return HashCodes.hashInt((int)(longBits ^ (longBits >>> 32)));
        case FLOAT:
            return HashCodes.hashInt(Float.floatToRawIntBits(((Float)key).floatValue()));
        default:
            throw new IllegalArgumentException("Unknown field type: " + fieldType);
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory.encoding;

import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.LongStream;

/**
 * This class stores multiple instances of a fixed bit width element in a list of nodes. For
 * example, it can store multiple 6 bit elements at different indices of the list of nodes.
 *
 * Under the hood, it uses a {@link FixedLengthElementArray} in order to implement compact storage
 * that allows inserting multiple instances of the fixed length elements. It maintains at least
 * enough space to hold up to the max number of occurrences of each element. This means that if
 * there are 8 elements at one node index, it will contain enough space to store at least 8 elements
 * at each index.
 *
 * Running out of space at any index triggers a relatively expensive resize operation, where we
 * create storage of a multiple (currently 1.5x) of the previous storage and copy items over, but
 * this can be alleviated by passing in a better guess for max elements per node.
 * Note that this class is currently designed to be used for a relatively small number for
 * bitsPerElement - it will not work for bitsPerElement greater than 60.
 */
public class FixedLengthMultipleOccurrenceElementArray {
    private static final double RESIZE_MULTIPLE = 1.5;
    private static final long NO_ELEMENT = 0L;

    private final ArraySegmentRecycler memoryRecycler;
    private final FixedLengthElementArray nodesWithOrdinalZero;
    private final int bitsPerElement;
    private final long elementMask;
    private final long numNodes;

    private volatile FixedLengthElementArray storage;
    private volatile int maxElementsPerNode;

    public FixedLengthMultipleOccurrenceElementArray(ArraySegmentRecycler memoryRecycler,
            long numNodes, int bitsPerElement, int maxElementsPerNodeEstimate) {
        nodesWithOrdinalZero = new FixedLengthElementArray(memoryRecycler, numNodes);
        storage = new FixedLengthElementArray(
                memoryRecycler, numNodes * bitsPerElement * maxElementsPerNodeEstimate);
        this.memoryRecycler = memoryRecycler;
        this.bitsPerElement = bitsPerElement;
        this.elementMask = (1L << bitsPerElement) - 1;
        this.numNodes = numNodes;
        this.maxElementsPerNode = maxElementsPerNodeEstimate;
    }

    /**
     * This method adds an element at nodeIndex. Note that this does not check for duplicates; if
     * the element already exists, another instance of it will be added.
     * This method is not thread-safe - you cannot call this method concurrently with itself or with
     * {@link #getElements}.
     *
     * @param nodeIndex the node index
     * @param element the element to add
     */
    public void addElement(long nodeIndex, long element) {
        if (element > elementMask) {
            throw new IllegalArgumentException("Element " + element + " does not fit in "
                    + bitsPerElement + " bits");
        }
        if (nodeIndex >= numNodes) {
            throw new IllegalArgumentException("Provided nodeIndex  " + nodeIndex
                    + " greater then numNodes " + numNodes);
        }
        if (element == NO_ELEMENT) {
            // we use 0 to indicate an "empty" element, so we have to store ordinal zero here
            nodesWithOrdinalZero.setElementValue(nodeIndex, 1, 1);
            return;
        }
        long bucketStart = nodeIndex * maxElementsPerNode * bitsPerElement;
        long currentIndex;
        int offset = 0;
        do {
            currentIndex = bucketStart + offset * bitsPerElement;
            offset++;
        } while (storage.getElementValue(currentIndex, bitsPerElement, elementMask) != NO_ELEMENT
                && offset < maxElementsPerNode);
        if (storage.getElementValue(currentIndex, bitsPerElement, elementMask) != NO_ELEMENT) {
            // we're full at this index - resize, then figure out the new current index
            resizeStorage();
            currentIndex = nodeIndex * maxElementsPerNode * bitsPerElement + offset * bitsPerElement;
        }
        /* we're adding to the first empty spot from the beginning of the bucket - this is
         * preferable to adding at the end because we want our getElements method to be fast, and
         * it's okay for addElement to be comparatively slow */
        storage.setElementValue(currentIndex, bitsPerElement, element);
    }

    /**
     * Return a list of elements at the specified node index. The returned list may contain
     * duplicates.
     * This method not thread-safe - the caller must ensure that no one calls {@link #addElement}
     * concurrently with this method, but calling this method concurrently with itself is safe.
     *
     * @param nodeIndex the node index
     * @return a list of element at the node index
     */
    public List<Long> getElements(long nodeIndex) {
        long bucketStart = nodeIndex * maxElementsPerNode * bitsPerElement;
        List<Long> ret = new ArrayList<>();
        if (nodesWithOrdinalZero.getElementValue(nodeIndex, 1, 1) != NO_ELEMENT) {
            // 0 indicates an "empty" element, so we fetch ordinal zeros from nodesWithOrdinalZero
            ret.add(NO_ELEMENT);
        }
        for (int offset = 0; offset < maxElementsPerNode; offset++) {
            long element = storage.getElementValue(bucketStart + offset * bitsPerElement,
                    bitsPerElement, elementMask);
            if (element == NO_ELEMENT) {
                break; // we have exhausted the elements at this index
            }
            ret.add(element);
        }
        return ret;
    }

    /**
     * A destructor function - call to free up the underlying memory.
     */
    public void destroy() {
        storage.destroy(memoryRecycler);
    }

    /**
     * Resize the underlying storage to a multiple of what it currently is. This method is not
     * thread-safe.
     */
    private void resizeStorage() {
        int currentElementsPerNode = maxElementsPerNode;
        int newElementsPerNode = (int) (currentElementsPerNode * RESIZE_MULTIPLE);
        if (newElementsPerNode <= currentElementsPerNode) {
            throw new IllegalStateException("cannot resize fixed length array from "
                    + currentElementsPerNode + " to " + newElementsPerNode);
        }
        FixedLengthElementArray newStorage = new FixedLengthElementArray(memoryRecycler,
                numNodes * bitsPerElement * newElementsPerNode);
        LongStream.range(0, numNodes).forEach(nodeIndex -> {
            long currentBucketStart = nodeIndex * currentElementsPerNode * bitsPerElement;
            long newBucketStart = nodeIndex * newElementsPerNode * bitsPerElement;
            for (int offset = 0; offset < currentElementsPerNode; offset++) {
                long element = storage.getElementValue(currentBucketStart + offset * bitsPerElement,
                        bitsPerElement, elementMask);
                if (element == NO_ELEMENT) {
                    break; // we have exhausted the elements at this index
                }
                newStorage.setElementValue(
                        newBucketStart + offset * bitsPerElement, bitsPerElement, element);
            }
        });
        storage.destroy(memoryRecycler);
        storage = newStorage;
        maxElementsPerNode = newElementsPerNode;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.jsonadapter;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.jsonadapter.chunker.JsonArrayChunker;
import com.netflix.hollow.jsonadapter.field.FieldProcessor;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

public abstract class AbstractHollowJsonAdaptorTask {
    public static boolean isDebug = false;

    protected final int maxWorkQueue = 2048;
    protected final SimultaneousExecutor executor = new SimultaneousExecutor(getClass(), "json-adaptor");

    protected final String typeName;
    protected final String actionName;
    protected final Map<String, Map<String, FieldProcessor>> fieldProcessors;

    public AbstractHollowJsonAdaptorTask(String typeName) {
        this(typeName, null);
    }

    public AbstractHollowJsonAdaptorTask(String typeName, String actionName) {
        this.typeName = typeName;
        this.actionName = actionName;
        this.fieldProcessors = new HashMap<String, Map<String,FieldProcessor>>();
    }

    public String getTypeName() {
        return typeName;
    }

    public void addFieldProcessor(FieldProcessor... processors) {
        for (FieldProcessor p : processors) {
            Map<String, FieldProcessor> entityFieldProcessors = fieldProcessors.get(p.getEntityName());
            if(entityFieldProcessors == null) {
                entityFieldProcessors = new HashMap<String, FieldProcessor>();
                fieldProcessors.put(p.getEntityName(), entityFieldProcessors);
            }

            entityFieldProcessors.put(p.getFieldName(), p);
        }
    }

    public FieldProcessor getFieldProcessor(String entityName, String fieldName) {
        Map<String, FieldProcessor> entityFieldProcessors = fieldProcessors.get(entityName);
        if(entityFieldProcessors == null)
            return null;
        return entityFieldProcessors.get(fieldName);
    }

    protected void processFile(File f, int maxSample) throws Exception {
        processFile(new FileReader(f), maxSample);
    }

    ///TODO: Many parse failures can cause out of memory errors.
    protected void processFile(Reader r, int maxSample) throws Exception {
        JsonArrayChunker chunker = new JsonArrayChunker(r, executor);
        chunker.initialize();

        int counter = 0;

        Reader jsonObj = chunker.nextChunk();
        while(jsonObj != null && counter < maxSample) {

            final Reader currentObject = jsonObj;

            executor.execute(new Runnable() {
                public void run() {
                    try {
                        JsonFactory factory = new JsonFactory();
                        JsonParser parser = factory.createParser(currentObject);
                        processRecord(parser);
                    } catch(Exception e){
                        throw new RuntimeException(e);
                    }
                }
            });

            while(executor.getQueue().size() > maxWorkQueue) {
                Thread.sleep(5);
            }

            counter++;

            jsonObj.close();
            jsonObj = chunker.nextChunk();
        }

        executor.awaitSuccessfulCompletion();
    }

    protected abstract int processRecord(JsonParser parser) throws IOException;

    protected boolean wait(List<Future<?>> futureList) throws Exception {
        boolean isSuccess = false;
        for (final Future<?> f : futureList) {
            try {
                f.get();
                isSuccess = true;
            } catch (final InterruptedException e) {
                e.printStackTrace();
            } catch (final ExecutionException e) {
                e.printStackTrace();
                throw e;
            }
        }

        return isSuccess;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;
import java.util.Arrays;
import java.util.BitSet;
import java.util.concurrent.atomic.AtomicLongArray;

/**
 * This data structure maps byte sequences to ordinals.  This is a hash table.
 *

 * The pointersAndOrdinals AtomicLongArray contains keys, and the {@link ByteDataBuffer}
 * contains values.  Each key has two components.
 *

 * The high 29 bits in the key represents the ordinal.  The low 35 bits represents the pointer to the start position
 * of the byte sequence in the ByteDataBuffer.  Each byte sequence is preceded by a variable-length integer
 * (see {@link VarInt}), indicating the length of the sequence.

 *
 * @author dkoszewnik
 */
public class ByteArrayOrdinalMap {

    private static final long EMPTY_BUCKET_VALUE = -1L;

    private static final int BITS_PER_ORDINAL = 29;
    private static final int BITS_PER_POINTER = Long.SIZE - BITS_PER_ORDINAL;
    private static final long POINTER_MASK = (1L << BITS_PER_POINTER) - 1;
    private static final long ORDINAL_MASK = (1L << BITS_PER_ORDINAL) - 1;
    private static final long MAX_BYTE_DATA_LENGTH = 1L << BITS_PER_POINTER;

    /// Thread safety:  We need volatile access semantics to the individual elements in the
    /// pointersAndOrdinals array.
    /// Ordinal is the high 29 bits.  Pointer to byte data is the low 35 bits.
    /// In addition need volatile access to the reference when resize occurs
    private volatile AtomicLongArray pointersAndOrdinals;
    private final ByteDataBuffer byteData;
    private final FreeOrdinalTracker freeOrdinalTracker;
    private int size;
    private int sizeBeforeGrow;

    private BitSet unusedPreviousOrdinals;

    private long[] pointersByOrdinal;

    /**
     * Creates a byte array ordinal map with a an initial capacity of 256 elements,
     * and a load factor of 70%.
     */
    public ByteArrayOrdinalMap() {
        this(256);
    }

    /**
     * Creates a byte array ordinal map with an initial capacity of a given size
     * rounded up to the nearest power of two, and a load factor of 70%.
     */
    public ByteArrayOrdinalMap(int size) {
        size = bucketSize(size);

        this.freeOrdinalTracker = new FreeOrdinalTracker();
        this.byteData = new ByteDataBuffer(WastefulRecycler.DEFAULT_INSTANCE);
        this.pointersAndOrdinals = emptyKeyArray(size);
        this.sizeBeforeGrow = (int) (((float) size) * 0.7); /// 70% load factor
        this.size = 0;
    }

    private static int bucketSize(int x) {
        // See Hackers Delight Fig. 3-3
        x = x - 1;
        x = x | (x >> 1);
        x = x | (x >> 2);
        x = x | (x >> 4);
        x = x | (x >> 8);
        x = x | (x >> 16);
        return (x < 256) ? 256 : (x >= 1 << 30) ? 1 << 30 : x + 1;
    }

    public int getOrAssignOrdinal(ByteDataBuffer serializedRepresentation) {
        return getOrAssignOrdinal(serializedRepresentation, -1);
    }

    /**
     * Adds a sequence of bytes to this map.  If the sequence of bytes has previously been added
     * to this map then its assigned ordinal is returned.
     * If the sequence of bytes has not been added to this map then a new ordinal is assigned
     * and returned.
     *

     * This operation is thread-safe.
     *
     * @param serializedRepresentation the sequence of bytes
     * @param preferredOrdinal the preferred ordinal to assign, if not already assigned to
     * another sequence of bytes and the given sequence of bytes has not previously been added
     * @return the assigned ordinal
     */
    public int getOrAssignOrdinal(ByteDataBuffer serializedRepresentation, int preferredOrdinal) {
        int hash = HashCodes.hashCode(serializedRepresentation);

        int ordinal = get(serializedRepresentation, hash);
        return ordinal != -1 ? ordinal : assignOrdinal(serializedRepresentation, hash, preferredOrdinal);
    }

    /// acquire the lock before writing.
    private synchronized int assignOrdinal(ByteDataBuffer serializedRepresentation, int hash, int preferredOrdinal) {
        if (preferredOrdinal < -1 || preferredOrdinal > ORDINAL_MASK) {
            throw new IllegalArgumentException(String.format(
                    "The given preferred ordinal %s is out of bounds and not within the closed interval [-1, %s]",
                    preferredOrdinal, ORDINAL_MASK));
        }
        if (size > sizeBeforeGrow) {
            growKeyArray();
        }

        /// check to make sure that after acquiring the lock, the element still does not exist.
        /// this operation is akin to double-checked locking which is 'fixed' with the JSR 133 memory model in JVM >= 1.5.
        /// Note that this also requires pointersAndOrdinals be volatile so resizes are also visible
        AtomicLongArray pao = pointersAndOrdinals;

        int modBitmask = pao.length() - 1;
        int bucket = hash & modBitmask;
        long key = pao.get(bucket);

        while (key != EMPTY_BUCKET_VALUE) {
            if (compare(serializedRepresentation, key)) {
                return (int) (key >>> BITS_PER_POINTER);
            }

            bucket = (bucket + 1) & modBitmask;
            key = pao.get(bucket);
        }

        /// the ordinal for this object still does not exist in the list, even after the lock has been acquired.
        /// it is up to this thread to add it at the current bucket position.
        int ordinal = findFreeOrdinal(preferredOrdinal);
        if (ordinal > ORDINAL_MASK) {
            throw new IllegalStateException(String.format(
                    "Ordinal cannot be assigned. The to be assigned ordinal, %s, is greater than the maximum supported ordinal value of %s",
                    ordinal, ORDINAL_MASK));
        }

        long pointer = byteData.length();

        VarInt.writeVInt(byteData, (int) serializedRepresentation.length());
        /// Copying might cause a resize to the segmented array held by byteData
        /// A reading thread may observe a null value for a segment during the creation
        /// of a new segments array (see SegmentedByteArray.ensureCapacity).
        serializedRepresentation.copyTo(byteData);
        if (byteData.length() > MAX_BYTE_DATA_LENGTH) {
            throw new IllegalStateException(String.format(
                    "The number of bytes for the serialized representations, %s, is too large and is greater than the maximum of %s bytes",
                    byteData.length(), MAX_BYTE_DATA_LENGTH));
        }

        key = ((long) ordinal << BITS_PER_POINTER) | pointer;

        size++;

        /// this set on the AtomicLongArray has volatile semantics (i.e. behaves like a monitor release).
        /// Any other thread reading this element in the AtomicLongArray will have visibility to all memory writes this thread has made up to this point.
        /// This means the entire byte sequence is guaranteed to be visible to any thread which reads the pointer to that data.
        pao.set(bucket, key);

        return ordinal;
    }

    /**
     * If the preferredOrdinal has not already been used, mark it and use it.  Otherwise,
     * delegate to the FreeOrdinalTracker.
     */
    private int findFreeOrdinal(int preferredOrdinal) {
        if (preferredOrdinal != -1 && unusedPreviousOrdinals.get(preferredOrdinal)) {
            unusedPreviousOrdinals.clear(preferredOrdinal);
            return preferredOrdinal;
        }

        return freeOrdinalTracker.getFreeOrdinal();
    }

    /**
     * Assign a predefined ordinal to a serialized representation.

     *

     * WARNING: THIS OPERATION IS NOT THREAD-SAFE.

     * WARNING: THIS OPERATION WILL NOT UPDATE THE FreeOrdinalTracker.
     *
     * @param serializedRepresentation the serialized representation
     * @param ordinal the ordinal
     */
    public void put(ByteDataBuffer serializedRepresentation, int ordinal) {
        if (ordinal < 0 || ordinal > ORDINAL_MASK) {
            throw new IllegalArgumentException(String.format(
                    "The given ordinal %s is out of bounds and not within the closed interval [0, %s]",
                    ordinal, ORDINAL_MASK));
        }
        if (size > sizeBeforeGrow) {
            growKeyArray();
        }

        int hash = HashCodes.hashCode(serializedRepresentation);

        AtomicLongArray pao = pointersAndOrdinals;

        int modBitmask = pao.length() - 1;
        int bucket = hash & modBitmask;
        long key = pao.get(bucket);

        while (key != EMPTY_BUCKET_VALUE) {
            bucket = (bucket + 1) & modBitmask;
            key = pao.get(bucket);
        }

        long pointer = byteData.length();

        VarInt.writeVInt(byteData, (int) serializedRepresentation.length());
        serializedRepresentation.copyTo(byteData);
        if (byteData.length() > MAX_BYTE_DATA_LENGTH) {
            throw new IllegalStateException(String.format(
                    "The number of bytes for the serialized representations, %s, is too large and is greater than the maximum of %s bytes",
                    byteData.length(), MAX_BYTE_DATA_LENGTH));
        }

        key = ((long) ordinal << BITS_PER_POINTER) | pointer;

        size++;

        pao.set(bucket, key);
    }

    public void recalculateFreeOrdinals() {
        BitSet populatedOrdinals = new BitSet();
        AtomicLongArray pao = pointersAndOrdinals;

        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                int ordinal = (int) (key >>> BITS_PER_POINTER);
                populatedOrdinals.set(ordinal);
            }
        }

        recalculateFreeOrdinals(populatedOrdinals);
    }

    public void reservePreviouslyPopulatedOrdinals(BitSet populatedOrdinals) {
        unusedPreviousOrdinals = BitSet.valueOf(populatedOrdinals.toLongArray());

        recalculateFreeOrdinals(populatedOrdinals);
    }

    private void recalculateFreeOrdinals(BitSet populatedOrdinals) {
        freeOrdinalTracker.reset();

        int length = populatedOrdinals.length();
        int ordinal = populatedOrdinals.nextClearBit(0);

        while (ordinal < length) {
            freeOrdinalTracker.returnOrdinalToPool(ordinal);
            ordinal = populatedOrdinals.nextClearBit(ordinal + 1);
        }

        freeOrdinalTracker.setNextEmptyOrdinal(length);
    }

    public BitSet getUnusedPreviousOrdinals() {
        return unusedPreviousOrdinals;
    }

    /**
     * Returns the ordinal for a previously added byte sequence.  If this byte sequence has not been added to the map, then -1 is returned.

     *

     * This is intended for use in the client-side heap-safe double snapshot load.
     *
     * @param serializedRepresentation the serialized representation
     * @return The ordinal for this serialized representation, or -1.
     */
    public int get(ByteDataBuffer serializedRepresentation) {
        return get(serializedRepresentation, HashCodes.hashCode(serializedRepresentation));
    }

    private int get(ByteDataBuffer serializedRepresentation, int hash) {
        AtomicLongArray pao = pointersAndOrdinals;

        int modBitmask = pao.length() - 1;
        int bucket = hash & modBitmask;
        long key = pao.get(bucket);

        // Linear probing to resolve collisions
        // Given the load factor it is guaranteed that the loop will terminate
        // as there will be at least one empty bucket
        // To ensure this is the case it is important that pointersAndOrdinals
        // is read into a local variable and thereafter used, otherwise a concurrent
        // size increase may break this invariant
        while (key != EMPTY_BUCKET_VALUE) {
            if (compare(serializedRepresentation, key)) {
                return (int) (key >>> BITS_PER_POINTER);
            }

            bucket = (bucket + 1) & modBitmask;
            key = pao.get(bucket);
        }

        return -1;
    }

    /**
     * Create an array mapping the ordinals to pointers, so that they can be easily looked up
     * when writing to blob streams.
     */
    public void prepareForWrite() {
        int maxOrdinal = 0;
        AtomicLongArray pao = pointersAndOrdinals;

        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                int ordinal = (int) (key >>> BITS_PER_POINTER);
                if (ordinal > maxOrdinal) {
                    maxOrdinal = ordinal;
                }
            }
        }

        long[] pbo = new long[maxOrdinal + 1];
        Arrays.fill(pbo, -1);

        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                int ordinal = (int) (key >>> BITS_PER_POINTER);
                pbo[ordinal] = key & POINTER_MASK;
            }
        }

        pointersByOrdinal = pbo;
    }

    /**
     * Reclaim space in the byte array used in the previous cycle, but not referenced in this cycle.

     *

     * This is achieved by shifting all used byte sequences down in the byte array, then updating
     * the key array to reflect the new pointers and exclude the removed entries.  This is also where ordinals
     * which are unused are returned to the pool.

     *
     * @param usedOrdinals a bit set representing the ordinals which are currently referenced by any image.
     */
    public void compact(ThreadSafeBitSet usedOrdinals) {
        long[] populatedReverseKeys = new long[size];

        int counter = 0;
        AtomicLongArray pao = pointersAndOrdinals;

        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                populatedReverseKeys[counter++] = key << BITS_PER_ORDINAL | key >>> BITS_PER_POINTER;
            }
        }

        Arrays.sort(populatedReverseKeys);

        SegmentedByteArray arr = byteData.getUnderlyingArray();
        long currentCopyPointer = 0;

        for (int i = 0; i < populatedReverseKeys.length; i++) {
            int ordinal = (int) (populatedReverseKeys[i] & ORDINAL_MASK);

            if (usedOrdinals.get(ordinal)) {
                long pointer = populatedReverseKeys[i] >>> BITS_PER_ORDINAL;
                int length = VarInt.readVInt(arr, pointer);
                length += VarInt.sizeOfVInt(length);

                if (currentCopyPointer != pointer) {
                    arr.copy(arr, pointer, currentCopyPointer, length);
                }

                populatedReverseKeys[i] = populatedReverseKeys[i] << BITS_PER_POINTER | currentCopyPointer;

                currentCopyPointer += length;
            } else {
                freeOrdinalTracker.returnOrdinalToPool(ordinal);
                populatedReverseKeys[i] = EMPTY_BUCKET_VALUE;
            }
        }

        byteData.setPosition(currentCopyPointer);
        freeOrdinalTracker.sort();

        // Reset the array then fill with compacted values
        // Volatile store not required, could use plain store
        // See VarHandles for JDK >= 9
        for (int i = 0; i < pao.length(); i++) {
            pao.lazySet(i, EMPTY_BUCKET_VALUE);
        }
        populateNewHashArray(pao, populatedReverseKeys);
        size = usedOrdinals.cardinality();

        pointersByOrdinal = null;
        unusedPreviousOrdinals = null;
    }

    public long getPointerForData(int ordinal) {
        long pointer = pointersByOrdinal[ordinal] & POINTER_MASK;
        return pointer + VarInt.nextVLongSize(byteData.getUnderlyingArray(), pointer);
    }

    public boolean isReadyForWriting() {
        return pointersByOrdinal != null;
    }

    public boolean isReadyForAddingObjects() {
        return pointersByOrdinal == null;
    }

    public long getDataSize() {
        return byteData.length();
    }

    public int maxOrdinal() {
        int maxOrdinal = -1;
        AtomicLongArray pao = pointersAndOrdinals;

        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                int ordinal = (int) (key >>> BITS_PER_POINTER);
                if (ordinal > maxOrdinal) {
                    maxOrdinal = ordinal;
                }
            }
        }
        return maxOrdinal;
    }

    /**
     * Compare the byte sequence contained in the supplied ByteDataBuffer with the
     * sequence contained in the map pointed to by the specified key, byte by byte.
     */
    private boolean compare(ByteDataBuffer serializedRepresentation, long key) {
        long position = key & POINTER_MASK;

        int sizeOfData = VarInt.readVInt(byteData.getUnderlyingArray(), position);

        if (sizeOfData != serializedRepresentation.length()) {
            return false;
        }

        position += VarInt.sizeOfVInt(sizeOfData);

        for (int i = 0; i < sizeOfData; i++) {
            if (serializedRepresentation.get(i) != byteData.get(position++)) {
                return false;
            }
        }

        return true;
    }

    /**
     * Resize the ordinal map by increasing its capacity.
     *

     * No action is take if the current capacity is sufficient for the given size.
     *

     * WARNING: THIS OPERATION IS NOT THREAD-SAFE.
     *
     * @param size the size to increase to, rounded up to the nearest power of two.
     */
    public void resize(int size) {
        size = bucketSize(size);

        if (pointersAndOrdinals.length() < size) {
            growKeyArray(size);
        }
    }

    /**
     * Grow the key array.  All of the values in the current array must be re-hashed and added to the new array.
     */
    private void growKeyArray() {
        int newSize = pointersAndOrdinals.length() << 1;
        if (newSize < 0) {
            throw new IllegalStateException("New size computed to grow the underlying array for the map is negative. " +
                    "This is most likely due to the total number of keys added to map has exceeded the max capacity of the keys map can hold. "
                    +
                    "Current array size :" + pointersAndOrdinals.length() + " and size to grow :" + newSize);
        }
        growKeyArray(newSize);
    }

    private void growKeyArray(int newSize) {
        AtomicLongArray pao = pointersAndOrdinals;
        assert (newSize & (newSize - 1)) == 0; // power of 2
        assert pao.length() < newSize;

        AtomicLongArray newKeys = emptyKeyArray(newSize);

        long[] valuesToAdd = new long[size];

        int counter = 0;

        /// do not iterate over these values in the same order in which they appear in the hashed array.
        /// if we do so, we cause large clusters of collisions to appear (because we resolve collisions with linear probing).
        for (int i = 0; i < pao.length(); i++) {
            long key = pao.get(i);
            if (key != EMPTY_BUCKET_VALUE) {
                valuesToAdd[counter++] = key;
            }
        }

        Arrays.sort(valuesToAdd);

        populateNewHashArray(newKeys, valuesToAdd, counter);

        /// 70% load factor
        sizeBeforeGrow = (int) (((float) newSize) * 0.7);
        pointersAndOrdinals = newKeys;
    }

    /**
     * Hash all of the existing values specified by the keys in the supplied long array
     * into the supplied AtomicLongArray.
     */
    private void populateNewHashArray(AtomicLongArray newKeys, long[] valuesToAdd) {
        populateNewHashArray(newKeys, valuesToAdd, valuesToAdd.length);
    }

    private void populateNewHashArray(AtomicLongArray newKeys, long[] valuesToAdd, int length) {
        assert length <= valuesToAdd.length;

        int modBitmask = newKeys.length() - 1;

        for (int i = 0; i < length; i++) {
            long value = valuesToAdd[i];
            if (value != EMPTY_BUCKET_VALUE) {
                int hash = rehashPreviouslyAddedData(value);
                int bucket = hash & modBitmask;
                while (newKeys.get(bucket) != EMPTY_BUCKET_VALUE) {
                    bucket = (bucket + 1) & modBitmask;
                }
                // Volatile store not required, could use plain store
                // See VarHandles for JDK >= 9
                newKeys.lazySet(bucket, value);
            }
        }
    }

    /**
     * Get the hash code for the byte array pointed to by the specified key.
     */
    private int rehashPreviouslyAddedData(long key) {
        long position = key & POINTER_MASK;

        int sizeOfData = VarInt.readVInt(byteData.getUnderlyingArray(), position);
        position += VarInt.sizeOfVInt(sizeOfData);

        return HashCodes.hashCode(byteData.getUnderlyingArray(), position, sizeOfData);
    }

    /**
     * Create an AtomicLongArray of the specified size, each value in the array will be EMPTY_BUCKET_VALUE
     */
    private AtomicLongArray emptyKeyArray(int size) {
        AtomicLongArray arr = new AtomicLongArray(size);
        // Volatile store not required, could use plain store
        // See VarHandles for JDK >= 9
        for (int i = 0; i < arr.length(); i++) {
            arr.lazySet(i, EMPTY_BUCKET_VALUE);
        }
        return arr;
    }

    public ByteDataBuffer getByteData() {
        return byteData;
    }

    public AtomicLongArray getPointersAndOrdinals() {
        return pointersAndOrdinals;
    }

    public static boolean isPointerAndOrdinalEmpty(long pointerAndOrdinal) {
        return pointerAndOrdinal == EMPTY_BUCKET_VALUE;
    }

    public static long getPointer(long pointerAndOrdinal) {
        return pointerAndOrdinal & POINTER_MASK;
    }

    public static int getOrdinal(long pointerAndOrdinal) {
        return (int) (pointerAndOrdinal >>> BITS_PER_POINTER);
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index.traversal;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.dataaccess.HollowDataAccess;
import com.netflix.hollow.core.read.dataaccess.HollowObjectTypeDataAccess;
import com.netflix.hollow.core.read.dataaccess.HollowTypeDataAccess;
import com.netflix.hollow.core.util.IntList;
import java.util.BitSet;

/**
 * Used by the HollowHashIndex to traverse the possible value combinations of individual records given
 * a set of field paths.
 */
public class HollowIndexerValueTraverser {

    private final String fieldPaths[];
    private final HollowIndexerTraversalNode rootNode;
    private final IntList fieldMatchLists[];
    private final HollowTypeDataAccess fieldTypeDataAccess[];
    private final int fieldSchemaPosition[];

    public HollowIndexerValueTraverser(HollowDataAccess dataAccess, String type, String... fieldPaths) {
        this.fieldPaths = fieldPaths;

        TraversalTreeBuilder builder = new TraversalTreeBuilder(dataAccess, type, fieldPaths);

        this.rootNode = builder.buildTree();
        this.fieldMatchLists = builder.getFieldMatchLists();
        this.fieldTypeDataAccess = builder.getFieldTypeDataAccesses();
        this.fieldSchemaPosition = builder.getFieldSchemaPositions();
    }

    public void traverse(int ordinal) {
        for(int i=0;i<fieldMatchLists.length;i++)
            fieldMatchLists[i].clear();

        rootNode.traverse(ordinal);
    }

    public int getNumFieldPaths() {
        return fieldPaths.length;
    }

    public String getFieldPath(int idx) {
        return fieldPaths[idx];
    }

    public int getNumMatches() {
        return fieldMatchLists[0].size();
    }

    public Object getMatchedValue(int matchIdx, int fieldIdx) {
        int matchedOrdinal = fieldMatchLists[fieldIdx].get(matchIdx);
        return HollowReadFieldUtils.fieldValueObject((HollowObjectTypeDataAccess)fieldTypeDataAccess[fieldIdx], matchedOrdinal, fieldSchemaPosition[fieldIdx]);
    }

    public boolean isMatchedValueEqual(int matchIdx, int fieldIdx, Object value) {
        int matchedOrdinal = fieldMatchLists[fieldIdx].get(matchIdx);
        return HollowReadFieldUtils.fieldValueEquals((HollowObjectTypeDataAccess)fieldTypeDataAccess[fieldIdx], matchedOrdinal, fieldSchemaPosition[fieldIdx], value);
    }

    public int getMatchHash(int matchIdx) {
        int hashCode = 0;
        for(int i=0;i<getNumFieldPaths();i++) {
            hashCode ^= HashCodes.hashInt(HollowReadFieldUtils.fieldHashCode((HollowObjectTypeDataAccess)fieldTypeDataAccess[i], fieldMatchLists[i].get(matchIdx), fieldSchemaPosition[i]));
            hashCode ^= HashCodes.hashInt(hashCode);
        }
        return hashCode;
    }

    public int getMatchHash(int matchIdx, BitSet fields) {
        int hashCode = 0;
        for(int i=0;i<getNumFieldPaths();i++) {
            if(fields.get(i)) {
                hashCode ^= HashCodes.hashInt(HollowReadFieldUtils.fieldHashCode((HollowObjectTypeDataAccess)fieldTypeDataAccess[i], fieldMatchLists[i].get(matchIdx), fieldSchemaPosition[i]));
                hashCode ^= HashCodes.hashInt(hashCode);
            }
        }
        return hashCode;
    }

    /**
     * This method assumes the other traverser has the same match fields specified in the same order.
     *
     * @param matchIdx the match index
     * @param otherTraverser the other traverser
     * @param otherMatchIdx the other match index
     * @return true if this and the other traverser are equal
     */
    public boolean isMatchEqual(int matchIdx, HollowIndexerValueTraverser otherTraverser, int otherMatchIdx) {
        for(int i=0;i<getNumFieldPaths();i++) {
            if(!HollowReadFieldUtils.fieldsAreEqual((HollowObjectTypeDataAccess)fieldTypeDataAccess[i], fieldMatchLists[i].get(matchIdx), fieldSchemaPosition[i],
                    (HollowObjectTypeDataAccess)otherTraverser.fieldTypeDataAccess[i], otherTraverser.fieldMatchLists[i].get(otherMatchIdx), otherTraverser.fieldSchemaPosition[i]))
                return false;
        }
        return true;
    }

    public boolean isMatchEqual(int matchIdx, HollowIndexerValueTraverser otherTraverser, int otherMatchIdx, BitSet fields) {
        for(int i=0;i<getNumFieldPaths();i++) {
            if(fields.get(i)) {
                if(!HollowReadFieldUtils.fieldsAreEqual((HollowObjectTypeDataAccess)fieldTypeDataAccess[i], fieldMatchLists[i].get(matchIdx), fieldSchemaPosition[i],
                        (HollowObjectTypeDataAccess)otherTraverser.fieldTypeDataAccess[i], otherTraverser.fieldMatchLists[i].get(otherMatchIdx), otherTraverser.fieldSchemaPosition[i]))
                    return false;
            }
        }
        return true;
    }

    public int getMatchOrdinal(int matchIdx, int fieldIdx) {
        return fieldMatchLists[fieldIdx].get(matchIdx);
    }

    public HollowTypeDataAccess getFieldTypeDataAccess(int fieldIdx) {
        return fieldTypeDataAccess[fieldIdx];
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine.object;

import com.netflix.hollow.api.sampling.DisabledSamplingDirector;
import com.netflix.hollow.api.sampling.HollowObjectSampler;
import com.netflix.hollow.api.sampling.HollowSampler;
import com.netflix.hollow.api.sampling.HollowSamplingDirector;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.read.dataaccess.HollowObjectTypeDataAccess;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.SnapshotPopulatedOrdinalsReader;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.tools.checksum.HollowChecksum;
import java.io.DataInputStream;
import java.io.IOException;
import java.util.BitSet;

/**
 * A {@link HollowTypeReadState} for OBJECT type records.
 */
public class HollowObjectTypeReadState extends HollowTypeReadState implements HollowObjectTypeDataAccess {

    private final HollowObjectSchema unfilteredSchema;
    private final HollowObjectSampler sampler;

    private final int shardNumberMask;
    private final int shardOrdinalShift;
    private final HollowObjectTypeReadStateShard shards[];

    private int maxOrdinal;

    public HollowObjectTypeReadState(HollowReadStateEngine stateEngine, HollowObjectSchema schema) {
        this(stateEngine, schema, schema, 1);
    }

    public HollowObjectTypeReadState(HollowReadStateEngine stateEngine, HollowObjectSchema schema, HollowObjectSchema unfilteredSchema, int numShards) {
        super(stateEngine, schema);
        this.sampler = new HollowObjectSampler(schema, DisabledSamplingDirector.INSTANCE);
        this.unfilteredSchema = unfilteredSchema;
        this.shardNumberMask = numShards - 1;
        this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(numShards);

        if(numShards < 1 || 1 << shardOrdinalShift != numShards)
            throw new IllegalArgumentException("Number of shards must be a power of 2!");

        HollowObjectTypeReadStateShard shards[] = new HollowObjectTypeReadStateShard[numShards];
        for(int i=0;i<shards.length;i++)
            shards[i] = new HollowObjectTypeReadStateShard(schema);

        this.shards = shards;
    }

    @Override
    public HollowObjectSchema getSchema() {
        return (HollowObjectSchema)schema;
    }

    @Override
    public int maxOrdinal() {
        return maxOrdinal;
    }

    @Override
    public void readSnapshot(DataInputStream dis, ArraySegmentRecycler memoryRecycler) throws IOException {
        if(shards.length > 1)
            maxOrdinal = VarInt.readVInt(dis);

        for(int i=0;i<shards.length;i++) {
            HollowObjectTypeDataElements snapshotData = new HollowObjectTypeDataElements(getSchema(), memoryRecycler);
            snapshotData.readSnapshot(dis, unfilteredSchema);
            shards[i].setCurrentData(snapshotData);
        }

        if(shards.length == 1)
            maxOrdinal = shards[0].currentDataElements().maxOrdinal;

        SnapshotPopulatedOrdinalsReader.readOrdinals(dis, stateListeners);
    }

    @Override
    public void applyDelta(DataInputStream dis, HollowSchema deltaSchema, ArraySegmentRecycler memoryRecycler) throws IOException {
        if(shards.length > 1)
            maxOrdinal = VarInt.readVInt(dis);

        for(int i=0;i<shards.length;i++) {
            HollowObjectTypeDataElements deltaData = new HollowObjectTypeDataElements((HollowObjectSchema)deltaSchema, memoryRecycler);
            HollowObjectTypeDataElements nextData = new HollowObjectTypeDataElements(getSchema(), memoryRecycler);
            deltaData.readDelta(dis);
            HollowObjectTypeDataElements oldData = shards[i].currentDataElements();
            nextData.applyDelta(oldData, deltaData);
            shards[i].setCurrentData(nextData);
            notifyListenerAboutDeltaChanges(deltaData.encodedRemovals, deltaData.encodedAdditions, i, shards.length);
            deltaData.destroy();
            oldData.destroy();
            stateEngine.getMemoryRecycler().swap();
        }

        if(shards.length == 1)
            maxOrdinal = shards[0].currentDataElements().maxOrdinal;
    }

    public static void discardSnapshot(DataInputStream dis, HollowObjectSchema schema, int numShards) throws IOException {
        discardType(dis, schema, numShards, false);
    }

    public static void discardDelta(DataInputStream dis, HollowObjectSchema schema, int numShards) throws IOException {
        discardType(dis, schema, numShards, true);
    }

    public static void discardType(DataInputStream dis, HollowObjectSchema schema, int numShards, boolean delta) throws IOException {
        HollowObjectTypeDataElements.discardFromStream(dis, schema, numShards, delta);
        if(!delta)
            SnapshotPopulatedOrdinalsReader.discardOrdinals(dis);
    }

    @Override
    public boolean isNull(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].isNull(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public int readOrdinal(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readOrdinal(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public int readInt(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readInt(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public float readFloat(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readFloat(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public double readDouble(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readDouble(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public long readLong(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readLong(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public Boolean readBoolean(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readBoolean(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public byte[] readBytes(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readBytes(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public String readString(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].readString(ordinal >> shardOrdinalShift, fieldIndex);
    }

    @Override
    public boolean isStringFieldEqual(int ordinal, int fieldIndex, String testValue) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].isStringFieldEqual(ordinal >> shardOrdinalShift, fieldIndex, testValue);
    }

    @Override
    public int findVarLengthFieldHashCode(int ordinal, int fieldIndex) {
        sampler.recordFieldAccess(fieldIndex);
        return shards[ordinal & shardNumberMask].findVarLengthFieldHashCode(ordinal >> shardOrdinalShift, fieldIndex);
    }

    /**
     * Warning:  Not thread-safe.  Should only be called within the update thread.
     * @param fieldName the field name
     * @return the number of bits required for the field
     */
    public int bitsRequiredForField(String fieldName) {
        int maxBitsRequiredForField = shards[0].bitsRequiredForField(fieldName);

        for(int i=1;i<shards.length;i++) {
            int shardRequiredBits = shards[i].bitsRequiredForField(fieldName);
            if(shardRequiredBits > maxBitsRequiredForField)
                maxBitsRequiredForField = shardRequiredBits;
        }

        return maxBitsRequiredForField;
    }

    @Override
    public HollowSampler getSampler() {
        return sampler;
    }

    @Override
    protected void invalidate() {
        stateListeners = EMPTY_LISTENERS;
        for(int i=0;i<shards.length;i++)
            shards[i].invalidate();
    }

    @Override
    public void setSamplingDirector(HollowSamplingDirector director) {
        sampler.setSamplingDirector(director);
    }

    @Override
    public void setFieldSpecificSamplingDirector(HollowFilterConfig fieldSpec, HollowSamplingDirector director) {
        sampler.setFieldSpecificSamplingDirector(fieldSpec, director);
    }

    @Override
    public void ignoreUpdateThreadForSampling(Thread t) {
        sampler.setUpdateThread(t);
    }

    HollowObjectTypeDataElements[] currentDataElements() {
        HollowObjectTypeDataElements currentDataElements[] = new HollowObjectTypeDataElements[shards.length];

        for(int i=0;i<shards.length;i++)
            currentDataElements[i] = shards[i].currentDataElements();

        return currentDataElements;
    }

    @Override
    protected void applyToChecksum(HollowChecksum checksum, HollowSchema withSchema) {
        if(!(withSchema instanceof HollowObjectSchema))
            throw new IllegalArgumentException("HollowObjectTypeReadState can only calculate checksum with a HollowObjectSchema: " + getSchema().getName());

        BitSet populatedOrdinals = getPopulatedOrdinals();

        for(int i=0;i<shards.length;i++)
            shards[i].applyToChecksum(checksum, withSchema, populatedOrdinals, i, shards.length);
    }

	@Override
	public long getApproximateHeapFootprintInBytes() {
	    long totalApproximateHeapFootprintInBytes = 0;

	    for(int i=0;i<shards.length;i++)
	        totalApproximateHeapFootprintInBytes += shards[i].getApproximateHeapFootprintInBytes();

	    return totalApproximateHeapFootprintInBytes;
	}

	@Override
	public long getApproximateHoleCostInBytes() {
	    long totalApproximateHoleCostInBytes = 0;

	    BitSet populatedOrdinals = getPopulatedOrdinals();

	    for(int i=0;i<shards.length;i++)
	        totalApproximateHoleCostInBytes += shards[i].getApproximateHoleCostInBytes(populatedOrdinals, i, shards.length);

	    return totalApproximateHoleCostInBytes;
	}

	void setCurrentData(HollowObjectTypeDataElements data) {
	    if(shards.length > 1)
	        throw new UnsupportedOperationException("Cannot directly set data on sharded type state");
	    shards[0].setCurrentData(data);
	    maxOrdinal = data.maxOrdinal;
	}

    @Override
    public int numShards() {
        return shards.length;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.consumer.index;

import static java.util.stream.Collectors.toList;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.api.objects.HollowObject;
import com.netflix.hollow.core.index.FieldPaths;
import com.netflix.hollow.core.index.HollowPrimaryKeyIndex;
import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Stream;

/**
 * A type safe index for indexing with a unique key (such as a primary key).
 * <p>
 * If the index is {@link HollowConsumer#addRefreshListener(HollowConsumer.RefreshListener) registered} with its
 * associated {@link HollowConsumer} then the index will track updates and changes will be reflected in matched results
 * (performed after such updates).  When a registered index is no longer needed it should be
 * {@link HollowConsumer#removeRefreshListener(HollowConsumer.RefreshListener) deregistered} to avoid unnecessary
 * index recalculation and to ensure the index is reclaimed by the garbage collector.
 *
 * @param <T> the unique type
 * @param <Q> the key type
 */
public class UniqueKeyIndex<T extends HollowObject, Q>
        implements HollowConsumer.RefreshListener, HollowConsumer.RefreshRegistrationListener, Function<Q, T> {
    final HollowConsumer consumer;
    HollowAPI api;
    final SelectFieldPathResultExtractor<T> uniqueTypeExtractor;
    final List<MatchFieldPathArgumentExtractor<Q>> matchFields;
    final String uniqueTypeName;
    final String[] matchFieldPaths;
    HollowPrimaryKeyIndex hpki;

    UniqueKeyIndex(
            HollowConsumer consumer,
            Class<T> uniqueType,
            PrimaryKey primaryTypeKey,
            List<MatchFieldPathArgumentExtractor<Q>> matchFields) {
        this.consumer = consumer;
        this.api = consumer.getAPI();
        this.uniqueTypeName = uniqueType.getSimpleName();

        this.uniqueTypeExtractor = SelectFieldPathResultExtractor
                .from(consumer.getAPI().getClass(), consumer.getStateEngine(), uniqueType, "", uniqueType);

        if (primaryTypeKey != null) {
            matchFields = validatePrimaryKeyFieldPaths(consumer, uniqueTypeName, primaryTypeKey, matchFields);
        }

        this.matchFields = matchFields;
        this.matchFieldPaths = matchFields.stream()
                .map(mf -> mf.fieldPath.toString())
                .toArray(String[]::new);

        this.hpki = new HollowPrimaryKeyIndex(consumer.getStateEngine(), uniqueTypeName, matchFieldPaths);
    }

    static <Q> List<MatchFieldPathArgumentExtractor<Q>> validatePrimaryKeyFieldPaths(
            HollowConsumer consumer, String primaryTypeName,
            PrimaryKey primaryTypeKey, List<MatchFieldPathArgumentExtractor<Q>> matchFields) {

        // Validate primary key field paths
        List<FieldPaths.FieldPath<FieldPaths.ObjectFieldSegment>> paths = Stream.of(
                primaryTypeKey.getFieldPaths())
                .map(fp -> FieldPaths
                        .createFieldPathForPrimaryKey(consumer.getStateEngine(), primaryTypeName, fp))
                .collect(toList());

        // Validate that primary key field paths are the same as that on the match fields
        // If so then match field extractors are shuffled to have the same order as primary key field paths

        List<MatchFieldPathArgumentExtractor<Q>> orderedMatchFields = paths.stream().flatMap(
                path -> {
                    MatchFieldPathArgumentExtractor<Q> mfe =
                            matchFields.stream().filter(e -> e.fieldPath.equals(path)).findFirst().orElse(null);
                    return mfe != null ? Stream.of(mfe) : null;
                }).collect(toList());

        if (orderedMatchFields.size() != paths.size()) {
            // @@@
            throw new IllegalArgumentException();
        }
        return orderedMatchFields;
    }

    UniqueKeyIndex(
            HollowConsumer consumer,
            Class<T> uniqueType,
            PrimaryKey primaryTypeKey,
            Class<Q> matchFieldsType) {
        // @@@ Use FieldPaths.createFieldPathForPrimaryKey
        this(consumer,
                uniqueType,
                primaryTypeKey,
                MatchFieldPathArgumentExtractor
                        .fromHolderClass(consumer.getStateEngine(), uniqueType, matchFieldsType,
                                FieldPaths::createFieldPathForPrimaryKey));
    }

    UniqueKeyIndex(
            HollowConsumer consumer,
            Class<T> uniqueType,
            PrimaryKey primaryTypeKey,
            String fieldPath, Class<Q> matchFieldType) {
        // @@@ Use FieldPaths.createFieldPathForPrimaryKey
        this(consumer,
                uniqueType,
                primaryTypeKey,
                Collections.singletonList(
                        MatchFieldPathArgumentExtractor
                                .fromPathAndType(consumer.getStateEngine(), uniqueType, fieldPath, matchFieldType,
                                        FieldPaths::createFieldPathForPrimaryKey)));
    }

    /**
     * Finds the unique object, an instance of the unique type, for a given key.
     *
     * @param key the key
     * @return the unique object
     */
    @Override
    public T apply(Q key) {
        return findMatch(key);
    }

    /**
     * Finds the unique object, an instance of the unique type, for a given key.
     *
     * @param key the key
     * @return the unique object
     */
    public T findMatch(Q key) {
        Object[] keyArray = matchFields.stream().map(mf -> mf.extract(key)).toArray();

        int ordinal = hpki.getMatchingOrdinal(keyArray);
        if (ordinal == -1) {
            return null;
        }

        return uniqueTypeExtractor.extract(api, ordinal);
    }

    // HollowConsumer.RefreshListener

    @Override public void refreshStarted(long currentVersion, long requestedVersion) {
    }

    @Override public void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) {
        HollowPrimaryKeyIndex hpki = this.hpki;
        hpki.detachFromDeltaUpdates();
        hpki = new HollowPrimaryKeyIndex(consumer.getStateEngine(), hpki.getPrimaryKey());
        hpki.listenForDeltaUpdates();
        this.hpki = hpki;
        this.api = api;
    }

    @Override public void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) {
        this.api = api;
    }

    @Override public void blobLoaded(HollowConsumer.Blob transition) {
    }

    @Override public void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion) {
    }

    @Override public void refreshFailed(
            long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause) {
    }

    // HollowConsumer.RefreshRegistrationListener

    @Override public void onBeforeAddition(HollowConsumer c) {
        if (c != consumer) {
            throw new IllegalStateException("The index's consumer and the listener's consumer are not the same");
        }
        hpki.listenForDeltaUpdates();
    }

    @Override public void onAfterRemoval(HollowConsumer c) {
        hpki.detachFromDeltaUpdates();
    }

    /**
     * Starts the building of a {@link UniqueKeyIndex}.
     *
     * @param consumer the consumer containing instances of the given unique type
     * @param uniqueType the unique type
     * @param <T> the unique type
     * @return a builder
     */
    public static <T extends HollowObject> Builder<T> from(HollowConsumer consumer, Class<T> uniqueType) {
        Objects.requireNonNull(consumer);
        Objects.requireNonNull(uniqueType);
        return new Builder<>(consumer, uniqueType);
    }

    /**
     * The builder of a {@link UniqueKeyIndex}.
     *
     * @param <T> the unique type
     */
    public static final class Builder<T extends HollowObject> {
        final HollowConsumer consumer;
        final Class<T> uniqueType;
        PrimaryKey primaryTypeKey; // non-null on bindWithPrimaryKeyOnType

        Builder(HollowConsumer consumer, Class<T> uniqueType) {
            this.consumer = consumer;
            this.uniqueType = uniqueType;
        }

        /**
         * Binds the field paths with those of the primary key associated with the schema of the unique type.
         *
         * @throws com.netflix.hollow.api.error.SchemaNotFoundException if there is no schema for the unique
         * type
         * @throws IllegalArgumentException if there is no primary key associated with the unique type
         */
        public Builder<T> bindToPrimaryKey() {
            String primaryTypeName = uniqueType.getSimpleName();
            HollowSchema schema = consumer.getStateEngine().getNonNullSchema(primaryTypeName);

            assert schema.getSchemaType() == HollowSchema.SchemaType.OBJECT;

            this.primaryTypeKey = ((HollowObjectSchema) schema).getPrimaryKey();
            if (primaryTypeKey == null) {
                throw new IllegalArgumentException(
                        String.format("No primary key associated with primary type %s", uniqueType));
            }
            return this;
        }

        /**
         * Creates a {@link UniqueKeyIndex} for matching with field paths and types declared by
         * {@link FieldPath} annotated fields or methods on the given key type.
         *
         * @param keyType the key type
         * @param <Q> the key type
         * @return a {@code UniqueKeyIndex}
         * @throws IllegalArgumentException if the key type declares one or more invalid field paths
         * or invalid types given resolution of corresponding field paths
         * @throws IllegalArgumentException if the builder is bound to the primary key of the unique type and
         * the field paths declared by the key type are not the identical to those declared by the primary key
         */
        public <Q> UniqueKeyIndex<T, Q> usingBean(Class<Q> keyType) {
            Objects.requireNonNull(keyType);
            return new UniqueKeyIndex<>(consumer, uniqueType, primaryTypeKey,
                    keyType);
        }

        /**
         * Creates a {@link UniqueKeyIndex} for matching with a single key field path and type.
         *
         * @param keyFieldPath the key field path
         * @param keyFieldType the key type
         * @param <Q> the key type
         * @return a {@code UniqueKeyIndex}
         * @throws IllegalArgumentException if the key field path is empty or invalid
         * @throws IllegalArgumentException if the key field type is invalid given resolution of the
         * key field path
         * @throws IllegalArgumentException if the builder is bound to the primary key of the unique type and
         * the field path declared by the key type is not identical to the keyFieldPath
         */
        public <Q> UniqueKeyIndex<T, Q> usingPath(String keyFieldPath, Class<Q> keyFieldType) {
            Objects.requireNonNull(keyFieldPath);
            if (keyFieldPath.isEmpty()) {
                throw new IllegalArgumentException("keyFieldPath argument is an empty String");
            }
            Objects.requireNonNull(keyFieldType);
            return new UniqueKeyIndex<>(consumer, uniqueType, primaryTypeKey,
                    keyFieldPath, keyFieldType);
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.producer;

import com.netflix.hollow.api.producer.HollowProducer.ReadState;
import com.netflix.hollow.core.HollowConstants;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;

/**
 * Beta API subject to change.
 *
 * Helper for {@link HollowProducer} to manage current and pending read states and the
 * transition that occurs within a cycle.
 *
 * @author Tim Taylor {@literal<tim@toolbear.io>}
 *
 */
final class ReadStateHelper {
    static ReadStateHelper newDeltaChain() {
        return new ReadStateHelper(null, null);
    }

    static ReadStateHelper restored(ReadState state) {
        return new ReadStateHelper(state, null);
    }

    static ReadState newReadState(final long version, final HollowReadStateEngine stateEngine) {
        return new HollowProducer.ReadState() {
            @Override
            public long getVersion() {
                return version;
            }

            @Override
            public HollowReadStateEngine getStateEngine() {
                return stateEngine;
            }
        };
    }

    private final ReadState current;
    private final ReadState pending;

    private ReadStateHelper(ReadState current, ReadState pending) {
        this.current = current;
        this.pending = pending;
    }

    ReadStateHelper roundtrip(long version) {
        if(pending != null) throw new IllegalStateException();
        return new ReadStateHelper(this.current, newReadState(version, new HollowReadStateEngine()));
    }

    /**
     * Swap underlying state engines between current and pending while keeping the versions consistent;
     * used after delta integrity checks have altered the underlying state engines.
     *
     * @return
     */
    ReadStateHelper swap() {
        return new ReadStateHelper(newReadState(current.getVersion(), pending.getStateEngine()),
                newReadState(pending.getVersion(), current.getStateEngine()));
    }

    ReadStateHelper commit() {
        if(pending == null) throw new IllegalStateException();
        return new ReadStateHelper(this.pending, null);
    }

    ReadStateHelper rollback() {
        if(pending == null) throw new IllegalStateException();
        return new ReadStateHelper(newReadState(current.getVersion(), pending.getStateEngine()), null);
    }

    ReadState current() {
        return current;
    }

    boolean hasCurrent() {
        return current != null;
    }

    ReadState pending() {
        return pending;
    }

    long pendingVersion() {
        return pending != null ? pending.getVersion() : HollowConstants.VERSION_NONE;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.schema;

import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;
import java.io.IOException;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;

/**
 * Parses text representations of {@link HollowSchema}.
 * <p>
 * The text representations are the same format obtained via toString() on a HollowSchema.
 */
public class HollowSchemaParser {

    private static final Logger log = Logger.getLogger(HollowSchemaParser.class.getName());

    /**
     * Parse a collection of {@link HollowSchema}s from the provided Reader.
     *
     * @param reader the reader
     * @return the list of schema
     * @throws IOException if the schema cannot be parsed
     */
    public static List<HollowSchema> parseCollectionOfSchemas(Reader reader) throws IOException {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        configureTokenizer(tokenizer);
        List<HollowSchema> schemaList = new ArrayList<HollowSchema>();

        HollowSchema schema = parseSchema(tokenizer);
        while (schema != null) {
            schemaList.add(schema);
            schema = parseSchema(tokenizer);
        }

        return schemaList;
    }

    /**
     * Parse a collection of {@link HollowSchema}s from the provided String.
     *
     * @param schemas the schemas as a string
     * @return the list of schema
     * @throws IOException if the schema cannot be parsed
     */
    public static List<HollowSchema> parseCollectionOfSchemas(String schemas) throws IOException {
        return parseCollectionOfSchemas(new StringReader(schemas));
    }

    /**
     * Parse a single {@link HollowSchema} from the provided String.
     *
     * @param schema the schema as a string
     * @return the schema
     * @throws IOException if the schema cannot be parsed
     */
    public static HollowSchema parseSchema(String schema) throws IOException {
        StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(schema));
        configureTokenizer(tokenizer);
        return parseSchema(tokenizer);
    }

    private static HollowSchema parseSchema(StreamTokenizer tokenizer) throws IOException {
        int tok = tokenizer.nextToken();
        while(tok != StreamTokenizer.TT_WORD) {
            if(tok == StreamTokenizer.TT_EOF)
                return null;
            tok = tokenizer.nextToken();
        }

        String typeName = tokenizer.sval;

        tok = tokenizer.nextToken();
        if(tok == StreamTokenizer.TT_WORD) {
            if("List".equals(tokenizer.sval)) {
                return parseListSchema(typeName, tokenizer);
            } else if("Set".equals(tokenizer.sval)) {
                return parseSetSchema(typeName, tokenizer);
            } else if("Map".equals(tokenizer.sval)) {
                return parseMapSchema(typeName, tokenizer);
            } else {
                throw new IOException("Invalid syntax: expected one of '{', 'List', 'Set', or 'Map' after type declaration for '" + typeName + "'");
            }
        }

        return parseObjectSchema(typeName, tokenizer);
    }

    private static HollowObjectSchema parseObjectSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
        String keyFieldPaths[] = parsePrimaryKey(tokenizer);
        if (tokenizer.ttype != '{') {
            throw new IOException("Invalid syntax: expecting '{' for '" + typeName + "'");
        }

        int tok = tokenizer.nextToken();
        List<String> tokens = new ArrayList<String>();
        while(tokenizer.ttype != '}') {
            if(tok != StreamTokenizer.TT_WORD)
                throw new IOException("Invalid syntax, expected field type: " + typeName);

            tokens.add(tokenizer.sval);
            tokenizer.nextToken();

            if(tok != StreamTokenizer.TT_WORD)
                throw new IOException("Invalid syntax, expected field name: " + typeName);

            String fieldName = tokenizer.sval;
            tokens.add(fieldName);
            tokenizer.nextToken();

            if(tokenizer.ttype != ';')
                throw new IOException("Invalid syntax, expected semicolon: " + typeName + "." + fieldName);

            tokenizer.nextToken();
        }

        HollowObjectSchema schema = new HollowObjectSchema(typeName, tokens.size() / 2, keyFieldPaths);
        for(int i=0;i<tokens.size();i+=2) {
            String fieldType = tokens.get(i);

            if("int".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.INT);
            } else if("long".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.LONG);
            } else if("float".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.FLOAT);
            } else if("double".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.DOUBLE);
            } else if("boolean".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.BOOLEAN);
            } else if("string".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.STRING);
            } else if("bytes".equals(fieldType)) {
                schema.addField(tokens.get(i+1), FieldType.BYTES);
            } else {
                schema.addField(tokens.get(i+1), FieldType.REFERENCE, fieldType);
            }
        }

        return schema;
    }

    private static HollowListSchema parseListSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
        int tok = tokenizer.nextToken();

        if(tokenizer.ttype != '<')
            throw new IOException("Invalid Syntax: Expected '<' after 'List' for type " + typeName);

        tok = tokenizer.nextToken();
        if(tok != StreamTokenizer.TT_WORD) {
            log.warning("Invalid Syntax: Expected element type declaration: " + typeName);
        }

        String elementType = tokenizer.sval;

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != '>')
            throw new IOException("Invalid Syntax: Expected '>' element type declaration: " + typeName);

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != ';')
            throw new IOException("Invalid Syntax: Expected semicolon after List schema declaration: " + typeName);

        return new HollowListSchema(typeName, elementType);
    }

    private static HollowSetSchema parseSetSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
        int tok = tokenizer.nextToken();

        if(tokenizer.ttype != '<')
            throw new IOException("Invalid Syntax: Expected '<' after 'Set' for type " + typeName);

        tok = tokenizer.nextToken();
        if(tok != StreamTokenizer.TT_WORD) {
            log.warning("Invalid Syntax: Expected element type declaration: " + typeName);
        }

        String elementType = tokenizer.sval;

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != '>')
            throw new IOException("Invalid Syntax: Expected '>' element type declaration: " + typeName);

        tok = tokenizer.nextToken();
        String hashKeyPaths[] = parseHashKey(tokenizer);

        if(tokenizer.ttype != ';')
            throw new IOException("Invalid Syntax: Expected semicolon after Set schema declaration: " + typeName);

        return new HollowSetSchema(typeName, elementType, hashKeyPaths);
    }

    private static HollowMapSchema parseMapSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
        int tok = tokenizer.nextToken();

        if(tokenizer.ttype != '<')
            throw new IOException("Invalid Syntax: Expected '<' after 'Map' for type " + typeName);

        tok = tokenizer.nextToken();
        if(tok != StreamTokenizer.TT_WORD) {
            log.warning("Invalid Syntax: Expected element type declaration: " + typeName);
        }

        String keyType = tokenizer.sval;

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != ',')
            throw new IOException("Invalid Syntax: Expected ',' after key type declaration: " + typeName);

        tok = tokenizer.nextToken();
        if(tok != StreamTokenizer.TT_WORD) {
            log.warning("Invalid Syntax: Expected value type declaration: " + typeName);
        }

        String valueType = tokenizer.sval;

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != '>')
            throw new IOException("Invalid Syntax: Expected '>' after value type declaration: " + typeName);

        tok = tokenizer.nextToken();
        String hashKeyPaths[] = parseHashKey(tokenizer);

        if(tokenizer.ttype != ';')
            throw new IOException("Invalid Syntax: Expected semicolon after Map schema declaration: " + typeName);

        return new HollowMapSchema(typeName, keyType, valueType, hashKeyPaths);
    }

    private static String[] parseHashKey(StreamTokenizer tokenizer) throws IOException {
        return parseKeyFieldPaths(tokenizer, "HashKey");
    }

    private static String[] parsePrimaryKey(StreamTokenizer tokenizer) throws IOException {
        return parseKeyFieldPaths(tokenizer, "PrimaryKey");
    }

    private static String[] parseKeyFieldPaths(StreamTokenizer tokenizer, String annotationName) throws IOException {
        if(tokenizer.ttype != '@')
            return new String[0];

        List<String> fieldPaths = new ArrayList<String>();

        int tok = tokenizer.nextToken();
        if(tok != StreamTokenizer.TT_WORD || !annotationName.equals(tokenizer.sval)) {
            throw new IOException("Invalid Syntax: Invalid  @" + tokenizer.sval + " annotation, expecting @" + annotationName + " declaraction");
        }

        tok = tokenizer.nextToken();
        if(tokenizer.ttype != '(')
            throw new IOException("Expected open parenthesis '(' after @" + annotationName + " declaration");

        tok = tokenizer.nextToken();
        while(tokenizer.ttype != ')') {
            if(tok != StreamTokenizer.TT_WORD)
                throw new IOException("Invalid field declaration inside @" + annotationName + "spec");

            fieldPaths.add(tokenizer.sval);

            tok = tokenizer.nextToken();

            if(tokenizer.ttype == ',') {
                tok = tokenizer.nextToken();
            } else if(tokenizer.ttype != ')') {
                throw new IOException("Invalid char inside @" + annotationName + " spec");
            }
        }

        tok = tokenizer.nextToken();

        return fieldPaths.toArray(new String[fieldPaths.size()]);
    }

    private static void configureTokenizer(StreamTokenizer tokenizer) {
        tokenizer.wordChars('_', '_');
        tokenizer.slashSlashComments(true);
        tokenizer.slashStarComments(true);
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.write.objectmapper;

import com.netflix.hollow.core.memory.ByteDataBuffer;
import com.netflix.hollow.core.write.HollowTypeWriteState;
import com.netflix.hollow.core.write.HollowWriteRecord;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter;

import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
import java.util.Set;

public abstract class HollowTypeMapper {

    public static final long ASSIGNED_ORDINAL_CYCLE_MASK = 0xFFFFFFFF00000000L;

    private final ThreadLocal<HollowWriteRecord> writeRec = new ThreadLocal<>();

    private final ThreadLocal<ByteDataBuffer> flatRecBuffer = new ThreadLocal<>();

    protected abstract String getTypeName();

    protected abstract int write(Object obj);

    protected abstract int writeFlat(Object obj, FlatRecordWriter flatRecordWriter);

    protected abstract HollowWriteRecord newWriteRecord();

    protected abstract HollowTypeWriteState getTypeWriteState();

    protected void addTypeState(HollowWriteStateEngine stateEngine) {
        if(stateEngine.getTypeState(getTypeName()) == null)
            stateEngine.addTypeState(getTypeWriteState());
    }

    protected HollowWriteRecord writeRecord() {
        HollowWriteRecord rec = writeRec.get();
        if(rec == null) {
            rec = newWriteRecord();
            writeRec.set(rec);
        }
        rec.reset();
        return rec;
    }

    protected ByteDataBuffer flatRecBuffer() {
    	ByteDataBuffer buf = flatRecBuffer.get();
    	if(buf == null) {
    		buf = new ByteDataBuffer();
    		flatRecBuffer.set(buf);
    	}
    	buf.reset();
    	return buf;
    }

    /**
     * Calculates the type name from a given type.
     *

     * If the type is annotated with {@link HollowTypeName} then the type name
     * is the value of the {@code HollowTypeName.name} attribute. Otherwise
     * the type name is derived from the type itself.
     * If the type is a {@code Class} then the type name is the simple name of
     * that class.
     * If the type is a parameterized type and is assignable to a class of {@code List},
     * {@code Set}, or {@code Map} then the type name begins with the simple class name of
     * the parameterized type's raw type, followed by "Of", followed by the result of
     * calling this method with the associated parameterized types (in order, in-fixed by "To").
     * Otherwise, the type name is the simple class name of the parameterized type's raw type.
     *

     * The translation from type to type name is lossy since the simple class name of a class
     * is used.  This means that no two types, from different packages, but with the same simple
     * name can be utilized.
     *
     * @param type the type
     * @return the type name.
     */
    public static String getDefaultTypeName(Type type) {
        if(type instanceof Class) {
            Class<?> clazz = (Class<?>)type;
            HollowTypeName explicitTypeName = clazz.getAnnotation(HollowTypeName.class);
            if(explicitTypeName != null)
                return explicitTypeName.name();
            return clazz.getSimpleName();
        }

        ParameterizedType parameterizedType = (ParameterizedType)type;
        Class<?> clazz = (Class<?>)parameterizedType.getRawType();

        if(List.class.isAssignableFrom(clazz))
            return "ListOf" + getDefaultTypeName(parameterizedType.getActualTypeArguments()[0]);
        if(Set.class.isAssignableFrom(clazz))
            return "SetOf" + getDefaultTypeName(parameterizedType.getActualTypeArguments()[0]);
        if(Map.class.isAssignableFrom(clazz))
            return "MapOf" + getDefaultTypeName(parameterizedType.getActualTypeArguments()[0]) + "To" + getDefaultTypeName(parameterizedType.getActualTypeArguments()[1]);

        return clazz.getSimpleName();
    }

    protected long cycleSpecificAssignedOrdinalBits() {
        return getTypeWriteState().getStateEngine().getNextStateRandomizedTag() & ASSIGNED_ORDINAL_CYCLE_MASK;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory.encoding;

import static com.netflix.hollow.core.HollowConstants.HASH_TABLE_MAX_SIZE;

import com.netflix.hollow.core.memory.ArrayByteData;
import com.netflix.hollow.core.memory.ByteData;
import com.netflix.hollow.core.memory.ByteDataBuffer;

public class HashCodes {
    private static final int MURMURHASH_SEED = 0xeab524b9;

    public static int hashCode(ByteDataBuffer data) {
        return hashCode(data.getUnderlyingArray(), 0, (int) data.length());
    }

    public static int hashCode(final String data) {
        if(data == null)
            return -1;

        int arrayLen = calculateByteArrayLength(data);

        if(arrayLen == data.length()) {
            return hashCode(new ByteData() {
                @Override
                public byte get(long position) {
                    return (byte)(data.charAt((int)position) & 0x7F);
                }
            }, 0, data.length());
        } else {
            byte[] array = createByteArrayFromString(data, arrayLen);

            return hashCode(array);
        }
    }

    public static int hashCode(byte[] data) {
        return hashCode(new ArrayByteData(data), 0, data.length);
    }

    private static int calculateByteArrayLength(String data) {
        int length = data.length();
        for(int i=0;i<data.length();i++) {
            if(data.charAt(i) > 0x7F)
                length += VarInt.sizeOfVInt(data.charAt(i)) - 1;
        }
        return length;
    }

    private static byte[] createByteArrayFromString(String data, int arrayLen) {
        byte array[] = new byte[arrayLen];

        int pos = 0;
        for(int i=0;i<data.length();i++) {
            pos = VarInt.writeVInt(array, pos, data.charAt(i));
        }
        return array;
    }

    /**
     * MurmurHash3.  Adapted from:

     *
     * https://github.com/yonik/java_util/blob/master/src/util/hash/MurmurHash3.java

     *
     * On 11/19/2013 the license for this file read:

     *
     *  The MurmurHash3 algorithm was created by Austin Appleby.  This java port was authored by
     *  Yonik Seeley and is placed into the public domain.  The author hereby disclaims copyright
     *  to this source code.
     *

     *  This produces exactly the same hash values as the final C++
     *  version of MurmurHash3 and is thus suitable for producing the same hash values across
     *  platforms.
     *

     *  The 32 bit x86 version of this hash should be the fastest variant for relatively short keys like ids.
     *

     *  Note - The x86 and x64 versions do _not_ produce the same results, as the
     *  algorithms are optimized for their respective platforms.
     *

     *  See http://github.com/yonik/java_util for future updates to this file.
     *
     * @param data the data to hash
     * @param offset the offset
     * @param len the length
     * @return the hash code
     */
    public static int hashCode(ByteData data, long offset, int len) {

        final int c1 = 0xcc9e2d51;
        final int c2 = 0x1b873593;

        int h1 = MURMURHASH_SEED;
        long roundedEnd = offset + (len & 0xfffffffffffffffcL); // round down to
                                                                // 4 byte block

        for (long i = offset; i < roundedEnd; i += 4) {
            // little endian load order
            int k1 = (data.get(i) & 0xff) | ((data.get(i + 1) & 0xff) << 8) | ((data.get(i + 2) & 0xff) << 16) | (data.get(i + 3) << 24);
            k1 *= c1;
            k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
            k1 *= c2;

            h1 ^= k1;
            h1 = (h1 << 13) | (h1 >>> 19); // ROTL32(h1,13);
            h1 = h1 * 5 + 0xe6546b64;
        }

        // tail
        int k1 = 0;

        switch (len & 0x03) {
        case 3:
            k1 = (data.get(roundedEnd + 2) & 0xff) << 16;
            // fallthrough
        case 2:
            k1 |= (data.get(roundedEnd + 1) & 0xff) << 8;
            // fallthrough
        case 1:
            k1 |= (data.get(roundedEnd) & 0xff);
            k1 *= c1;
            k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
            k1 *= c2;
            h1 ^= k1;
        }

        // finalization
        h1 ^= len;

        // fmix(h1);
        h1 ^= h1 >>> 16;
        h1 *= 0x85ebca6b;
        h1 ^= h1 >>> 13;
        h1 *= 0xc2b2ae35;
        h1 ^= h1 >>> 16;

        return h1;
    }

    public static int hashLong(long key) {
        key = (~key) + (key << 18);
        key ^= (key >>> 31);
        key *= 21;
        key ^= (key >>> 11);
        key += (key << 6);
        key ^= (key >>> 22);
        return (int) key;
    }

    public static int hashInt(int key) {
        key = ~key + (key << 15);
        key = key ^ (key >>> 12);
        key = key + (key << 2);
        key = key ^ (key >>> 4);
        key = key * 2057;
        key = key ^ (key >>> 16);
        return key;
    }

    /**
     * Determine size of hash table capable of storing the specified number of elements with a load
     * factor applied.
     *
     * @param numElements number of elements to be stored in the table
     * @return size of hash table, always a power of 2
     * @throws IllegalArgumentException when numElements is negative or exceeds
     *                                  {@link com.netflix.hollow.core.HollowConstants#HASH_TABLE_MAX_SIZE}
     */
    public static int hashTableSize(int numElements) throws IllegalArgumentException {
        if (numElements < 0) {
            throw new IllegalArgumentException("cannot be negative; numElements="+numElements);
        } else if (numElements > HASH_TABLE_MAX_SIZE) {
            throw new IllegalArgumentException("exceeds maximum number of buckets; numElements="+numElements);
        }

        if (numElements == 0)
            return 1;
        if (numElements < 3)
            return numElements * 2;

        // Apply load factor to number of elements and determine next
        // largest power of 2 that fits in an int
        int sizeAfterLoadFactor = (int)((long)numElements * 10 / 7);
        int bits = 32 - Integer.numberOfLeadingZeros(sizeAfterLoadFactor - 1);
        return 1 << bits;
    }
}