/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 */
package com.netflix.hollow.api.codegen;

public class CodeGeneratorConfig {
    private String classPostfix = "";
    private String getterPrefix = "";
    private boolean useAggressiveSubstitutions = false;

    // @TODO: Need to default this to be true in next major version of Hollow
    private boolean usePackageGrouping = false;
    private boolean useBooleanFieldErgonomics = false;
    private boolean reservePrimaryKeyIndexForTypeWithPrimaryKey = false;
    private boolean useHollowPrimitiveTypes = false;
    private boolean restrictApiToFieldType = false;
    private boolean useVerboseToString = false;

    public CodeGeneratorConfig() {}

    public CodeGeneratorConfig(String classPostfix, String getterPrefix) {
        this.classPostfix = classPostfix;
        this.getterPrefix = getterPrefix;
    }

    // Make it easier to automatically use defaults for next major version
    public void initWithNextMajorVersionDefaults_V3() {
        usePackageGrouping = true;
        useBooleanFieldErgonomics = true;
        reservePrimaryKeyIndexForTypeWithPrimaryKey = true;
        useHollowPrimitiveTypes = true;
        restrictApiToFieldType = true;
        useVerboseToString = true;
    }

    public String getClassPostfix() {
        return classPostfix;
    }

    public void setClassPostfix(String classPostfix) {
        this.classPostfix = classPostfix;
    }

    public String getGetterPrefix() {
        return getterPrefix;
    }

    public void setGetterPrefix(String getterPrefix) {
        this.getterPrefix = getterPrefix;
    }

    public boolean isUsePackageGrouping() {
        return usePackageGrouping;
    }

    public void setUsePackageGrouping(boolean usePackageGrouping) {
        this.usePackageGrouping = usePackageGrouping;
    }

    public boolean isUseAggressiveSubstitutions() {
        return useAggressiveSubstitutions;
    }

    public void setUseAggressiveSubstitutions(boolean useAggressiveSubstitutions) {
        this.useAggressiveSubstitutions = useAggressiveSubstitutions;
    }

    public boolean isUseBooleanFieldErgonomics() {
        return useBooleanFieldErgonomics;
    }

    public void setUseBooleanFieldErgonomics(boolean useBooleanFieldErgonomics) {
        this.useBooleanFieldErgonomics = useBooleanFieldErgonomics;
    }

    public boolean isReservePrimaryKeyIndexForTypeWithPrimaryKey() {
        return reservePrimaryKeyIndexForTypeWithPrimaryKey;
    }

    public void setReservePrimaryKeyIndexForTypeWithPrimaryKey(boolean reservePrimaryKeyIndexForTypeWithPrimaryKey) {
        this.reservePrimaryKeyIndexForTypeWithPrimaryKey = reservePrimaryKeyIndexForTypeWithPrimaryKey;
    }

    public boolean isListenToDataRefresh() {
        return !reservePrimaryKeyIndexForTypeWithPrimaryKey; // NOTE: to be backwards compatible
    }

    public boolean isUseHollowPrimitiveTypes() {
        return useHollowPrimitiveTypes;
    }

    public void setUseHollowPrimitiveTypes(boolean useHollowPrimitiveTypes) {
        this.useHollowPrimitiveTypes = useHollowPrimitiveTypes;
    }

    public boolean isRestrictApiToFieldType() {
        return restrictApiToFieldType;
    }

    public void setRestrictApiToFieldType(boolean restrictApiToFieldType) {
        this.restrictApiToFieldType = restrictApiToFieldType;
    }

    public boolean isUseVerboseToString() {
        return useVerboseToString;
    }

    public void setUseVerboseToString(boolean useVerboseToString) {
        this.useVerboseToString = useVerboseToString;
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + ((classPostfix == null) ? 0 : classPostfix.hashCode());
        result = prime * result + ((getterPrefix == null) ? 0 : getterPrefix.hashCode());
        result = prime * result + (reservePrimaryKeyIndexForTypeWithPrimaryKey ? 1231 : 1237);
        result = prime * result + (restrictApiToFieldType ? 1231 : 1237);
        result = prime * result + (useAggressiveSubstitutions ? 1231 : 1237);
        result = prime * result + (useBooleanFieldErgonomics ? 1231 : 1237);
        result = prime * result + (useHollowPrimitiveTypes ? 1231 : 1237);
        result = prime * result + (usePackageGrouping ? 1231 : 1237);
        result = prime * result + (useVerboseToString ? 1231 : 1237);
        return result;
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClass() != obj.getClass())
            return false;
        CodeGeneratorConfig other = (CodeGeneratorConfig) obj;
        if (classPostfix == null) {
            if (other.classPostfix != null)
                return false;
        } else if (!classPostfix.equals(other.classPostfix))
            return false;
        if (getterPrefix == null) {
            if (other.getterPrefix != null)
                return false;
        } else if (!getterPrefix.equals(other.getterPrefix))
            return false;
        if (reservePrimaryKeyIndexForTypeWithPrimaryKey != other.reservePrimaryKeyIndexForTypeWithPrimaryKey)
            return false;
        if (restrictApiToFieldType != other.restrictApiToFieldType)
            return false;
        if (useAggressiveSubstitutions != other.useAggressiveSubstitutions)
            return false;
        if (useBooleanFieldErgonomics != other.useBooleanFieldErgonomics)
            return false;
        if (useHollowPrimitiveTypes != other.useHollowPrimitiveTypes)
            return false;
        if (usePackageGrouping != other.usePackageGrouping)
            return false;
        if (useVerboseToString != other.useVerboseToString)
            return false;
        return true;
    }

    @Override
    public String toString() {
        StringBuilder builder = new StringBuilder();
        builder.append("CodeGeneratorConfig [classPostfix=");
        builder.append(classPostfix);
        builder.append(", getterPrefix=");
        builder.append(getterPrefix);
        builder.append(", usePackageGrouping=");
        builder.append(usePackageGrouping);
        builder.append(", useAggressiveSubstitutions=");
        builder.append(useAggressiveSubstitutions);
        builder.append(", useBooleanFieldErgonomics=");
        builder.append(useBooleanFieldErgonomics);
        builder.append(", reservePrimaryKeyIndexForTypeWithPrimaryKey=");
        builder.append(reservePrimaryKeyIndexForTypeWithPrimaryKey);
        builder.append(", useHollowPrimitiveTypes=");
        builder.append(useHollowPrimitiveTypes);
        builder.append(", restrictApiToFieldType=");
        builder.append(restrictApiToFieldType);
        builder.append(", useVerboseToString=");
        builder.append(useVerboseToString);
        builder.append("]");
        return builder.toString();
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index;

import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import java.util.Arrays;

public class GrowingSegmentedLongArray {

    private long[][] segments;
    private final int log2OfSegmentSize;
    private final int bitmask;

    private final ArraySegmentRecycler memoryRecycler;

    public GrowingSegmentedLongArray(ArraySegmentRecycler memoryRecycler) {
        this.memoryRecycler = memoryRecycler;
        this.log2OfSegmentSize = memoryRecycler.getLog2OfLongSegmentSize();
        this.bitmask = (1 << log2OfSegmentSize) - 1;
        this.segments = new long[64][];
    }

    /**
     * Set the byte at the given index to the specified value
     * @param index the index
     * @param value the byte
     */
    public void set(long index, long value) {
        int segmentIndex = (int)(index >> log2OfSegmentSize);

        if(segmentIndex >= segments.length) {
            int nextPowerOfTwo = 1 << (32 - Integer.numberOfLeadingZeros(segmentIndex));
            segments = Arrays.copyOf(segments, nextPowerOfTwo);
        }

        if(segments[segmentIndex] == null) {
            segments[segmentIndex] = memoryRecycler.getLongArray();
        }

        int longInSegment = (int)(index & bitmask);
        segments[segmentIndex][longInSegment] = value;
    }

    /**
     * Get the value of the byte at the specified index.
     * @param index the index
     * @return the byte value
     */
    public long get(long index) {
        int segmentIndex = (int)(index >> log2OfSegmentSize);

        if(segmentIndex >= segments.length || segments[segmentIndex] == null)
            return 0;

        int longInSegment = (int)(index & bitmask);
        return segments[segmentIndex][longInSegment];
    }

    public void destroy() {
        for(int i=0;i<segments.length;i++) {
            if(segments[i] != null)
                memoryRecycler.recycleLongArray(segments[i]);
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.query;

import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;
import com.netflix.hollow.core.schema.HollowSchema.SchemaType;
import com.netflix.hollow.tools.traverse.TransitiveSetTraverser;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;

/**
 * A HollowFieldMatchQuery can be used to scan through all records in a dataset to match specific field name/value combinations.
 *

 * Results are returned in the form of a Map<String, BitSet>.  Each type for which any records matched will have an entry in the
 * returned Map, keyed by type name.  The corresponding value is a BitSet which is set at the positions of the ordinals of
 * the matched records.
 *

 * Hint: The returned Map<String, BitSet> may be in turn passed to the {@link TransitiveSetTraverser} to be augmented with any records
 * which reference matched records.  For example, we can imagine a data model for which the following code would provide a
 * selection which includes the Actor record for "Tom Hanks", plus any Movie records in which he stars:
 *

 * {@code
 * HollowFieldMatchQuery query = new HollowFieldMatchQuery(myStateEngine);
 * Map<String, BitSet> selection = query.findMatchingRecords("actorName", "Tom Hanks");
 *
 * TransitiveSetTraverser.addReferencingOutsideClosure(myStateEngine, selection);
 * }
 *

 */
public class HollowFieldMatchQuery {

    private final HollowReadStateEngine readEngine;

    public HollowFieldMatchQuery(HollowReadStateEngine readEngine) {
        this.readEngine = readEngine;
    }

    /**
     * Match any records which include a field with the provided fieldName and value.
     *
     * @param fieldName the field name
     * @param fieldValue the field value as a string that will be parsed as the type of the field to match.
     * @return the matching records
     */
    public Map<String, BitSet> findMatchingRecords(String fieldName, String fieldValue) {
        Map<String, BitSet> matches = new HashMap<String, BitSet>();

        for(HollowTypeReadState typeState : readEngine.getTypeStates()) {
            augmentMatchingRecords(typeState, fieldName, fieldValue, matches);
        }

        return matches;
    }

    /**
     * Match any records of the specified type, which have the specified field set to the specified value.
     *
     * @param typeName the type name
     * @param fieldName the field name
     * @param fieldValue the field value as a string that will be parsed as the type of the field to match.
     * @return the matching records
     */
    public Map<String, BitSet> findMatchingRecords(String typeName, String fieldName, String fieldValue) {
        Map<String, BitSet> matches = new HashMap<String, BitSet>();

        HollowTypeReadState typeState = readEngine.getTypeState(typeName);
        if(typeState != null)
            augmentMatchingRecords(typeState, fieldName, fieldValue, matches);

        return matches;
    }

    private void augmentMatchingRecords(HollowTypeReadState typeState, String fieldName, String fieldValue, Map<String, BitSet> matches) {
        if(typeState.getSchema().getSchemaType() == SchemaType.OBJECT) {
            HollowObjectSchema schema = (HollowObjectSchema)typeState.getSchema();

            for(int i=0;i<schema.numFields();i++) {
                if(schema.getFieldName(i).equals(fieldName)) {
                    HollowObjectTypeReadState objState = (HollowObjectTypeReadState)typeState;

                    BitSet typeQueryMatches = null;

                    if(schema.getFieldType(i) == FieldType.REFERENCE) {
                        typeQueryMatches = attemptReferenceTraversalQuery(objState, i, fieldValue);
                    } else {
                        Object queryValue = castQueryValue(fieldValue, schema.getFieldType(i));

                        if(queryValue != null) {
                            typeQueryMatches = queryBasedOnValueMatches(objState, i, queryValue);
                        }
                    }

                    if(typeQueryMatches != null && typeQueryMatches.cardinality() > 0)
                        matches.put(typeState.getSchema().getName(), typeQueryMatches);
                }
            }
        }
    }

    private BitSet attemptReferenceTraversalQuery(HollowObjectTypeReadState typeState, int fieldIdx, String fieldValue) {
        HollowTypeReadState referencedTypeState = typeState.getSchema().getReferencedTypeState(fieldIdx);

        if(referencedTypeState.getSchema().getSchemaType() == SchemaType.OBJECT) {
            HollowObjectTypeReadState refObjTypeState = (HollowObjectTypeReadState)referencedTypeState;
            HollowObjectSchema refSchema = refObjTypeState.getSchema();

            if(refSchema.numFields() == 1) {
                if(refSchema.getFieldType(0) == FieldType.REFERENCE) {
                    BitSet refQueryMatches = attemptReferenceTraversalQuery(refObjTypeState, 0, fieldValue);
                    if(refQueryMatches != null)
                        return queryBasedOnMatchedReferences(typeState, fieldIdx, refQueryMatches);
                } else {
                    Object queryValue = castQueryValue(fieldValue, refSchema.getFieldType(0));

                    if(queryValue != null) {
                        BitSet refQueryMatches = queryBasedOnValueMatches(refObjTypeState, 0, queryValue);
                        if(refQueryMatches.cardinality() > 0)
                            return queryBasedOnMatchedReferences(typeState, fieldIdx, refQueryMatches);
                    }
                }
            }
        }

        return null;
    }

    private BitSet queryBasedOnMatchedReferences(HollowObjectTypeReadState typeState, int referenceFieldPosition, BitSet matchedReferences) {
        BitSet populatedOrdinals = typeState.getPopulatedOrdinals();
        BitSet typeQueryMatches = new BitSet(populatedOrdinals.length());

        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            int refOrdinal = typeState.readOrdinal(ordinal, referenceFieldPosition);
            if(refOrdinal != -1 && matchedReferences.get(refOrdinal))
                typeQueryMatches.set(ordinal);
            ordinal = populatedOrdinals.nextSetBit(ordinal+1);
        }
        return typeQueryMatches;
    }

    private BitSet queryBasedOnValueMatches(HollowObjectTypeReadState typeState, int fieldPosition, Object queryValue) {
        BitSet populatedOrdinals = typeState.getPopulatedOrdinals();
        BitSet typeQueryMatches = new BitSet(populatedOrdinals.length());

        int ordinal = populatedOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            if(HollowReadFieldUtils.fieldValueEquals(typeState, ordinal, fieldPosition, queryValue))
                typeQueryMatches.set(ordinal);
            ordinal = populatedOrdinals.nextSetBit(ordinal+1);
        }
        return typeQueryMatches;
    }

    private Object castQueryValue(String fieldValue, FieldType fieldType) {

        try {
            switch(fieldType) {
            case BOOLEAN:
                return Boolean.valueOf(fieldValue);
            case DOUBLE:
                return Double.parseDouble(fieldValue);
            case FLOAT:
                return Float.parseFloat(fieldValue);
            case INT:
                return Integer.parseInt(fieldValue);
            case LONG:
                return Long.parseLong(fieldValue);
            case STRING:
                return fieldValue;
            default:
                return null;
            }
        } catch(Exception e) {
            return null;
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.diff.specific;

import com.netflix.hollow.core.index.traversal.HollowIndexerValueTraverser;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.tools.diff.HollowDiffMatcher;
import java.util.Arrays;
import java.util.BitSet;
import java.util.concurrent.atomic.AtomicLong;

/**
 * The HollowSpecificDiff allows for the investigation of diffs among specific fields for matched records in two states.
 */
public class HollowSpecificDiff {

    private final HollowReadStateEngine from;
    private final HollowReadStateEngine to;
    private final HollowDiffMatcher matcher;
    private final String type;

    private BitSet elementKeyPaths;
    private BitSet elementNonKeyPaths;
    private String elementPaths[];

    private final AtomicLong totalUnmatchedFromElements;
    private final AtomicLong totalUnmatchedToElements;
    private final AtomicLong totalModifiedElements;
    private final AtomicLong totalMatchedEqualElements;

    /**
     * @param from the from state
     * @param to the to state
     * @param type the type to diff
     */
    public HollowSpecificDiff(HollowReadStateEngine from, HollowReadStateEngine to, String type) {
        this.from = from;
        this.to = to;
        this.matcher = new HollowDiffMatcher((HollowObjectTypeReadState)from.getTypeState(type), (HollowObjectTypeReadState)to.getTypeState(type));
        this.type = type;
        this.totalUnmatchedFromElements = new AtomicLong();
        this.totalUnmatchedToElements = new AtomicLong();
        this.totalMatchedEqualElements = new AtomicLong();
        this.totalModifiedElements = new AtomicLong();
    }

    /**
     * Set the primary key paths which will be used to find matching records across the two states
     *
     * @param paths the key paths
     */
    public void setRecordMatchPaths(String... paths) {
        for(String path : paths)
            matcher.addMatchPath(path);
    }

    /**
     * Set the paths for which we will inspect differences across the two states
     *
     * @param paths the paths for inspection
     */
    public void setElementMatchPaths(String... paths) {
        resetResults();
        this.elementPaths = paths;
        this.elementKeyPaths = null;
        this.elementNonKeyPaths = null;
    }

    /**
     * Optionally specify paths for which we will match records within an individual type's hierarchy
     *
     * @param paths the paths for matching
     */
    public void setElementKeyPaths(String... paths) {
        resetResults();
        elementKeyPaths = new BitSet(elementPaths.length);
        for(int i=0;i<paths.length;i++) {
            int elementPathIdx = getElementPathIdx(paths[i]);
            if(elementPathIdx == -1)
                throw new IllegalArgumentException("Key path must have been specified as an element match path.  Offending path: " + paths[i]);
            elementKeyPaths.set(elementPathIdx);
        }

        elementNonKeyPaths = new BitSet(elementPaths.length);
        elementNonKeyPaths.set(0, elementPaths.length);
        elementNonKeyPaths.andNot(elementKeyPaths);
    }

    public int getElementPathIdx(String path) {
        for(int i=0;i<elementPaths.length;i++) {
            if(elementPaths[i].equals(path))
                return i;
        }
        return -1;
    }

    /**
     * Find the matching records (based on primary keys) across states
     */
    public void prepareMatches() {
        matcher.calculateMatches();
    }

    /**
     * Calculate the differences
     */
    public void calculate() {
        resetResults();
        SimultaneousExecutor executor = new SimultaneousExecutor(getClass(), "calculate");
        final int numThreads = executor.getCorePoolSize();

        for(int i=0;i<numThreads;i++) {
            final int threadNumber = i;
            executor.execute(new Runnable() {
                public void run() {
                    HollowIndexerValueTraverser fromTraverser = new HollowIndexerValueTraverser(from, type, elementPaths);
                    HollowIndexerValueTraverser toTraverser = new HollowIndexerValueTraverser(to, type, elementPaths);

                    int hashedResults[] = new int[16];

                    for(int i=threadNumber;i<matcher.getMatchedOrdinals().size();i += numThreads) {
                        long ordinalPair = matcher.getMatchedOrdinals().get(i);
                        int fromOrdinal = (int)(ordinalPair >>> 32);
                        int toOrdinal = (int)ordinalPair;

                        fromTraverser.traverse(fromOrdinal);
                        toTraverser.traverse(toOrdinal);

                        if(fromTraverser.getNumMatches() * 2 > hashedResults.length)
                            hashedResults = new int[hashTableSize(fromTraverser.getNumMatches())];

                        populateHashTable(fromTraverser, hashedResults);

                        countMatches(fromTraverser, toTraverser, hashedResults);
                    }

                    for(int i=threadNumber;i<matcher.getExtraInFrom().size();i+=numThreads) {
                        fromTraverser.traverse(matcher.getExtraInFrom().get(i));
                        totalUnmatchedFromElements.addAndGet(fromTraverser.getNumMatches());
                    }

                    for(int i=threadNumber;i<matcher.getExtraInTo().size();i+=numThreads) {
                        toTraverser.traverse(matcher.getExtraInTo().get(i));
                        totalUnmatchedToElements.addAndGet(toTraverser.getNumMatches());
                    }
                }
            });
        }

        try {
            executor.awaitSuccessfulCompletion();
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    private void countMatches(HollowIndexerValueTraverser fromTraverser, HollowIndexerValueTraverser toTraverser, int[] hashedResults) {
        int numMatchedEqualElements = 0;
        int numModifiedElements = 0;

        int hashMask = hashedResults.length - 1;

        for(int j=0;j<toTraverser.getNumMatches();j++) {
            int hash = elementKeyPaths == null ? toTraverser.getMatchHash(j) : toTraverser.getMatchHash(j, elementKeyPaths);
            int bucket = hash & hashMask;

            while(hashedResults[bucket] != -1) {
                if(elementKeyPaths == null) {
                    if(fromTraverser.isMatchEqual(hashedResults[bucket], toTraverser, j)) {
                        numMatchedEqualElements++;
                        break;
                    }
                } else {
                    if(fromTraverser.isMatchEqual(hashedResults[bucket], toTraverser, j, elementKeyPaths)) {
                        if(fromTraverser.isMatchEqual(hashedResults[bucket], toTraverser, j, elementNonKeyPaths))
                            numMatchedEqualElements++;
                        else
                            numModifiedElements++;
                        break;
                    }
                }

                bucket++;
                bucket &= hashMask;
            }
        }

        int numCommonMatches = numMatchedEqualElements + numModifiedElements;

        totalMatchedEqualElements.addAndGet(numMatchedEqualElements);
        totalModifiedElements.addAndGet(numModifiedElements);
        totalUnmatchedFromElements.addAndGet(fromTraverser.getNumMatches() - numCommonMatches);
        totalUnmatchedToElements.addAndGet(toTraverser.getNumMatches() - numCommonMatches);
    }

    private void populateHashTable(HollowIndexerValueTraverser fromTraverser, int[] hashedResults) {
        Arrays.fill(hashedResults, -1);
        int hashMask = hashedResults.length - 1;

        for(int j=0;j<fromTraverser.getNumMatches();j++) {
            int hash = elementKeyPaths == null ? fromTraverser.getMatchHash(j) : fromTraverser.getMatchHash(j, elementKeyPaths);
            int bucket = hash & hashMask;

            while(hashedResults[bucket] != -1) {
                bucket++;
                bucket &= hashMask;
            }

            hashedResults[bucket] = j;
        }
    }

    private int hashTableSize(int numMatches) {
        return 1 << (32 - Integer.numberOfLeadingZeros((numMatches * 2) - 1));
    }

    private void resetResults() {
        totalUnmatchedFromElements.set(0);
        totalUnmatchedToElements.set(0);
        totalMatchedEqualElements.set(0);
        totalModifiedElements.set(0);
    }

    public long getTotalUnmatchedFromElements() {
        return totalUnmatchedFromElements.get();
    }

    public long getTotalUnmatchedToElements() {
        return totalUnmatchedToElements.get();
    }

    public long getTotalMatchedEqualElements() {
        return totalMatchedEqualElements.get();
    }

    public long getTotalModifiedElements() {
        return totalModifiedElements.get();
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.diff.count;

import com.netflix.hollow.core.util.IntList;
import com.netflix.hollow.tools.diff.HollowDiff;
import com.netflix.hollow.tools.diff.HollowDiffNodeIdentifier;
import com.netflix.hollow.tools.diff.HollowTypeDiff;

/**
 * Obtained via a {@link HollowTypeDiff}, this is a report of the differences in a specific field between two data states.
 *
 */
public class HollowFieldDiff implements Comparable<HollowFieldDiff> {
    private final HollowDiffNodeIdentifier fieldIdentifier;
    private final IntList diffFromOrdinals;
    private final IntList diffToOrdinals;
    private final IntList diffPairScores;

    private long totalDiffScore;

    public HollowFieldDiff(HollowDiffNodeIdentifier fieldIdentifier) {
        this.diffFromOrdinals = new IntList();
        this.diffToOrdinals = new IntList();
        this.diffPairScores = new IntList();
        this.fieldIdentifier = fieldIdentifier;
    }

    /**
     * Should be called exclusively from the {@link HollowDiff} -- not intended for external consumption
     *
     * @param fromOrdinal the from ordinal
     * @param toOrdinal the to ordinal
     * @param score the score
     */
    public void addDiff(int fromOrdinal, int toOrdinal, int score) {
        if(isSameDiffAsLastAdd(fromOrdinal, toOrdinal)) {
            int scoreIdx = diffPairScores.size() - 1;
            diffPairScores.set(scoreIdx, diffPairScores.get(scoreIdx) + score);
        } else {
            diffFromOrdinals.add(fromOrdinal);
            diffToOrdinals.add(toOrdinal);
            diffPairScores.add(score);
        }

        totalDiffScore += score;
    }

    private boolean isSameDiffAsLastAdd(int fromOrdinal, int toOrdinal) {
        return diffFromOrdinals.size() > 0
                && diffFromOrdinals.get(diffFromOrdinals.size() - 1) == fromOrdinal
                && diffToOrdinals.get(diffToOrdinals.size() - 1) == toOrdinal;
    }

    /**
     * @return The identifier for the field on which this diff reports.
     */
    public HollowDiffNodeIdentifier getFieldIdentifier() {
        return fieldIdentifier;
    }

    /**
     * @return the total score, used to judge relative magnitude of the diff.
     */
    public long getTotalDiffScore() {
        return totalDiffScore;
    }

    /**
     * @return the number of records which had at least one diff for this field.
     */
    public int getNumDiffs() {
        return diffToOrdinals.size();
    }

    /**
     * @param diffPairIdx a number from 0-n, where n is the value returned from numDiffs
     * @return the from ordinal for the (diffPairIdx)th record pair in which there were differences for this field.
     */
    public int getFromOrdinal(int diffPairIdx) {
        return diffFromOrdinals.get(diffPairIdx);
    }

    /**
     * @param diffPairIdx a number from 0-n, where n is the value returned from numDiffs
     * @return the to ordinal for the (diffPairIdx)th record pair in which there were differences for this field.
     */
    public int getToOrdinal(int diffPairIdx) {
        return diffToOrdinals.get(diffPairIdx);
    }

    /**
     * @param diffPairIdx a number from 0-n, where n is the value returned from numDiffs
     * @return the score of the diff for this field in the (diffPairIdx)th record pair in which there were differences for this field.
     */
    public int getPairScore(int diffPairIdx) {
        return diffPairScores.get(diffPairIdx);
    }

    /**
     * This should be called exclusively from the {@link HollowDiff}.  Not for external consumption.
     * @param otherFieldDiff the field diff to add
     */
    public void addResults(HollowFieldDiff otherFieldDiff) {
        for(int i=0;i<otherFieldDiff.getNumDiffs();i++) {
            addDiff(otherFieldDiff.getFromOrdinal(i), otherFieldDiff.getToOrdinal(i), otherFieldDiff.getPairScore(i));
        }
    }

    /**
     * Comparison is based on the totalDiffScore().
     */
    @Override
    public int compareTo(HollowFieldDiff o) {
        if(o.getTotalDiffScore() > totalDiffScore)
            return 1;
        else if(o.getTotalDiffScore() < totalDiffScore)
            return -1;
        return 0;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.consumer;

import static com.netflix.hollow.core.util.Threads.daemonThread;
import static java.util.concurrent.Executors.newSingleThreadExecutor;

import com.netflix.hollow.api.client.FailedTransitionTracker;
import com.netflix.hollow.api.client.HollowAPIFactory;
import com.netflix.hollow.api.client.HollowClientUpdater;
import com.netflix.hollow.api.client.StaleHollowReferenceDetector;
import com.netflix.hollow.api.codegen.HollowAPIClassJavaGenerator;
import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.api.metrics.HollowConsumerMetrics;
import com.netflix.hollow.api.metrics.HollowMetricsCollector;
import com.netflix.hollow.core.HollowConstants;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.util.DefaultHashCodeFinder;
import com.netflix.hollow.core.util.HollowObjectHashCodeFinder;
import com.netflix.hollow.tools.history.HollowHistory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * A HollowConsumer is the top-level class used by consumers of Hollow data to initialize and keep up-to-date a local in-memory
 * copy of a hollow dataset.  The interactions between the "blob" transition store and announcement listener are defined by
 * this class, and the implementations of the data retrieval, announcement mechanism are abstracted in the interfaces which
 * are provided to this class.
 *

 * To obtain a HollowConsumer, you should use a builder pattern, for example:
 *

 * {@code
 *
 * HollowConsumer consumer = HollowConsumer.withBlobRetriever(retriever)
 *                                         .withAnnouncementWatcher(watcher)
 *                                         .withGeneratedAPIClass(MovieAPI.class)
 *                                         .build();
 * }
 *

 *

 * The following components are injectable, but only an implementation of the HollowConsumer.BlobRetriever is
 * required to be injected, all other components are optional. :
 *

 *
{@link HollowConsumer.BlobRetriever}

 *
Implementations of this class define how to retrieve blob data from the blob store.

 *
 *
{@link HollowConsumer.AnnouncementWatcher}

 *
Implementations of this class define the announcement mechanism, which is used to track the version of the
 * currently announced state.  It's also expected that implementations will trigger a refresh each time current
 * data version is updated.

 *
 *
a List of {@link HollowConsumer.RefreshListener}s

 *
RefreshListener implementations will define what to do when various events happen before, during, and after updating
 * local in-memory copies of hollow data sets.

 *
 *
the Class representing a generated Hollow API

 *
Defines how to create a {@link HollowAPI} for the dataset, useful when wrapping a dataset with an api which has
 * been generated (via the {@link HollowAPIClassJavaGenerator})

 *
 *
{@link HollowFilterConfig}

 *
Defines what types and fields to load (or not load) into memory from hollow datasets.  Generally useful to reduce
 * heap footprint on consumers which do not require visibility of an entire dataset.

 *
 *
{@link HollowConsumer.DoubleSnapshotConfig}

 *
Defines whether this consumer may attempt a double snapshot, and how many deltas will be attempted during a single refresh.
 * A double snapshot will allow your consumer to update in case of a broken delta chain, but will also result in a doubling of
 * the heap footprint while the double snapshot is occurring.

 *
 *
{@link HollowConsumer.ObjectLongevityConfig}

 *
Object longevity is used to guarantee that Hollow objects which are backed by removed records will remain usable and
 * consistent until old references are discarded.  This behavior is turned off by default.  Implementations of this config
 * can be used to enable and configure this behavior.

 *
 *
{@link HollowConsumer.ObjectLongevityDetector}

 *
Implementations of this config will be notified when usage of expired Hollow object references is attempted.

 *
 *
An Executor

 *
The Executor which will be used to perform updates when {@link #triggerAsyncRefresh()} is called.  This will
 * default to a new fixed thread pool with a single refresh thread.

 *
 *

 */
@SuppressWarnings({"unused", "WeakerAccess"})
public class HollowConsumer {
    private static final Logger LOG = Logger.getLogger(HollowConsumer.class.getName());

    protected final AnnouncementWatcher announcementWatcher;
    protected final HollowClientUpdater updater;
    protected final ReadWriteLock refreshLock;
    protected final HollowConsumerMetrics metrics;

    private final Executor refreshExecutor;

    protected HollowConsumer(BlobRetriever blobRetriever,
                             AnnouncementWatcher announcementWatcher,
                             List<RefreshListener> refreshListeners,
                             HollowAPIFactory apiFactory,
                             HollowFilterConfig dataFilter,
                             ObjectLongevityConfig objectLongevityConfig,
                             ObjectLongevityDetector objectLongevityDetector,
                             DoubleSnapshotConfig doubleSnapshotConfig,
                             HollowObjectHashCodeFinder hashCodeFinder,
                             Executor refreshExecutor) {
        this(blobRetriever, announcementWatcher, refreshListeners, apiFactory, dataFilter,
                objectLongevityConfig, objectLongevityDetector, doubleSnapshotConfig
                , hashCodeFinder, refreshExecutor, null);
    }

    protected HollowConsumer(BlobRetriever blobRetriever,
                             AnnouncementWatcher announcementWatcher,
                             List<RefreshListener> refreshListeners,
                             HollowAPIFactory apiFactory,
                             HollowFilterConfig dataFilter,
                             ObjectLongevityConfig objectLongevityConfig,
                             ObjectLongevityDetector objectLongevityDetector,
                             DoubleSnapshotConfig doubleSnapshotConfig,
                             HollowObjectHashCodeFinder hashCodeFinder,
                             Executor refreshExecutor,
                             HollowMetricsCollector<HollowConsumerMetrics> metricsCollector) {

        this.metrics = new HollowConsumerMetrics();
        this.updater = new HollowClientUpdater(blobRetriever,
                refreshListeners,
                apiFactory,
                doubleSnapshotConfig,
                hashCodeFinder,
                objectLongevityConfig,
                objectLongevityDetector,
                metrics,
                metricsCollector);
        updater.setFilter(dataFilter);
        this.announcementWatcher = announcementWatcher;
        this.refreshExecutor = refreshExecutor;
        this.refreshLock = new ReentrantReadWriteLock();
        if (announcementWatcher != null)
            announcementWatcher.subscribeToUpdates(this);
    }

    /**
     * Triggers a refresh to the latest version specified by the {@link HollowConsumer.AnnouncementWatcher}.
     * If already on the latest version, this operation is a no-op.
     *

     * If a {@link HollowConsumer.AnnouncementWatcher} is not present, this call trigger a refresh to the
     * latest version available in the blob store.
     *

     * This is a blocking call.
     */
    public void triggerRefresh() {
        refreshLock.writeLock().lock();
        try {
            updater.updateTo(announcementWatcher == null ? Long.MAX_VALUE : announcementWatcher.getLatestVersion());
        } catch (Error | RuntimeException e) {
            throw e;
        } catch (Throwable t) {
            throw new RuntimeException(t);
        } finally {
            refreshLock.writeLock().unlock();
        }
    }

    /**
     * Immediately triggers a refresh in a different thread to the latest version
     * specified by the {@link HollowConsumer.AnnouncementWatcher}. If already on
     * the latest version, this operation is a no-op.
     *

     * If a {@link HollowConsumer.AnnouncementWatcher} is not present, this call trigger a refresh to the
     * latest version available in the blob store.
     *

     * This is an asynchronous call.
     */
    public void triggerAsyncRefresh() {
        triggerAsyncRefreshWithDelay(0);
    }

    /**
     * Triggers async refresh after the specified number of milliseconds has passed.
     *

     * Any subsequent calls for async refresh will not begin until after the specified delay
     * has completed.
     *
     * @param delayMillis the delay, in millseconds, before triggering the refresh
     */
    public void triggerAsyncRefreshWithDelay(int delayMillis) {
        final long targetBeginTime = System.currentTimeMillis() + delayMillis;

        refreshExecutor.execute(() -> {
            try {
                long delay = targetBeginTime - System.currentTimeMillis();
                if (delay > 0)
                    Thread.sleep(delay);
            } catch (InterruptedException e) {
                // Interrupting, such as shutting down the executor pool,
                // cancels the trigger
                LOG.log(Level.INFO, "Async refresh interrupted before trigger, refresh cancelled", e);
                return;
            }

            try {
                triggerRefresh();
            } catch (Error | RuntimeException e) {
                // Ensure exceptions are propagated to the executor
                LOG.log(Level.SEVERE, "Async refresh failed", e);
                throw e;
            }
        });
    }

    /**
     * If a {@link HollowConsumer.AnnouncementWatcher} is not specified, then this method will attempt to update
     * to the specified version, and if the specified version does not exist then to a different version as specified
     * by functionality in the {@code BlobRetriever}.
     *

     * Otherwise, an UnsupportedOperationException will be thrown.
     *

     * This is a blocking call.
     *
     * @param version the version to refresh to
     */
    public void triggerRefreshTo(long version) {
        if (announcementWatcher != null)
            throw new UnsupportedOperationException("Cannot trigger refresh to specified version when a HollowConsumer.AnnouncementWatcher is present");

        try {
            updater.updateTo(version);
        } catch (Error | RuntimeException e) {
            throw e;
        } catch (Throwable t) {
            throw new RuntimeException(t);
        }
    }

    /**
     * @return the {@link HollowReadStateEngine} which is holding the underlying hollow dataset.
     */
    public HollowReadStateEngine getStateEngine() {
        return updater.getStateEngine();
    }

    /**
     * @return the current version of the dataset.  This is the unique identifier of the data's state.
     */
    public long getCurrentVersionId() {
        return updater.getCurrentVersionId();
    }

    /**
     * Returns a {@code CompletableFuture} that completes after the initial data load succeeds.
     *

     * Callers can use methods like {@link CompletableFuture#join()} or {@link CompletableFuture#get(long, TimeUnit)}
     * to block until the initial load is complete.
     *

     * A failure during the initial load will not cause the future to complete exceptionally; this allows
     * for a subsequent data version to eventually succeed.
     *

     * In a consumer without published or announced versions – or one that always fails the initial load – the future
     * will remain incomplete indefinitely.
     *
     * @return a future which, when completed, has a value set to the data version that was initially loaded
     */
    public CompletableFuture<Long> getInitialLoad() {
        return updater.getInitialLoad();
    }

    /**
     * @return the api which wraps the underlying dataset.
     */
    public HollowAPI getAPI() {
        return updater.getAPI();
    }

    /**
     * Equivalent to calling {@link #getAPI()} and casting to the specified API.
     *
     * @param apiClass the class of the API
     * @param <T> the type of the API
     * @return the API which wraps the underlying dataset
     */
    public <T extends HollowAPI> T getAPI(Class<T> apiClass) {
        return apiClass.cast(updater.getAPI());
    }

    /**
     * Will force a double snapshot refresh on the next update.
     */
    public void forceDoubleSnapshotNextUpdate() {
        updater.forceDoubleSnapshotNextUpdate();
    }

    /**
     * Clear any failed transitions from the {@link FailedTransitionTracker}, so that they may be reattempted when an update is triggered.
     */
    public void clearFailedTransitions() {
        updater.clearFailedTransitions();
    }

    /**
     * @return the number of failed snapshot transitions stored in the {@link FailedTransitionTracker}.
     */
    public int getNumFailedSnapshotTransitions() {
        return updater.getNumFailedSnapshotTransitions();
    }

    /**
     * @return the number of failed delta transitions stored in the {@link FailedTransitionTracker}.
     */
    public int getNumFailedDeltaTransitions() {
        return updater.getNumFailedDeltaTransitions();
    }

    /**
     * @return a {@link ReadWriteLock#readLock()}, the corresponding writeLock() of which is used to synchronize refreshes.
     *

     * This is useful if performing long-running operations which require a consistent view of the entire dataset in a
     * single data state, to guarantee that updates do not happen while the operation runs.
     */
    public Lock getRefreshLock() {
        return refreshLock.readLock();
    }

    /**
     * Adds a {@link RefreshListener} to this consumer.
     *

     * If the listener was previously added to this consumer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is added, concurrently, during the occurrence of a refresh then the listener will not receive
     * events until the next refresh.  The listener may also be removed concurrently.
     *

     * If the listener instance implements {@link RefreshRegistrationListener} then before the listener is added
     * the {@link RefreshRegistrationListener#onBeforeAddition} method is be invoked.  If that method throws an
     * exception then that exception will be thrown by this method and the listener will not be added.
     *
     * @param listener the refresh listener to add
     */
    public void addRefreshListener(RefreshListener listener) {
        updater.addRefreshListener(listener, this);
    }

    /**
     * Removes a {@link RefreshListener} from this consumer.
     *

     * If the listener was not previously added to this consumer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is removed, concurrently, during  the occurrence of a refresh then the listener will receive all
     * events for that refresh but not receive events for subsequent any refreshes.
     *

     * If the listener instance implements {@link RefreshRegistrationListener} then after the listener is removed
     * the {@link RefreshRegistrationListener#onAfterRemoval} method is be invoked.  If that method throws an
     * exception then that exception will be thrown by this method.
     *
     * @param listener the refresh listener to remove
     */
    public void removeRefreshListener(RefreshListener listener) {
        updater.removeRefreshListener(listener, this);
    }

    /**
     * @return the metrics for this consumer
     */
    public HollowConsumerMetrics getMetrics() {
        return metrics;
    }

    /**
     * An interface which defines the necessary interactions of Hollow with a blob data store.
     *

     * Implementations will define how to retrieve blob data from a data store.
     */
    public interface BlobRetriever {

        /**
         * Returns the snapshot for the state with the greatest version identifier which is equal to or less than the desired version
         * @param desiredVersion the desired version
         * @return the blob of the snapshot
         */
        HollowConsumer.Blob retrieveSnapshotBlob(long desiredVersion);

        /**
         * Returns a delta transition which can be applied to the specified version identifier
         * @param currentVersion the current version
         * @return the blob of the delta
         */
        HollowConsumer.Blob retrieveDeltaBlob(long currentVersion);

        /**
         * Returns a reverse delta transition which can be applied to the specified version identifier
         * @param currentVersion the current version
         * @return the blob of the reverse delta
         */
        HollowConsumer.Blob retrieveReverseDeltaBlob(long currentVersion);
    }

    /**
     * A Blob, which is either a snapshot or a delta, defines three things:
     *

     *
The "from" version

     *
The unique identifier of the state to which a delta transition should be applied.  If
     * this is a snapshot, then this value is HollowConstants.VERSION_NONE.

     *
     *
The "to" version

     *
The unique identifier of the state at which a dataset will arrive after this blob is applied.

     *
     *
The actual blob data

     *
Implementations will define how to retrieve the actual blob data for this specific blob from a data store as an InputStream.

     *

     */
    public static abstract class Blob {

        private final long fromVersion;
        private final long toVersion;
        private final BlobType blobType;

        /**
         * Instantiate a snapshot to a specified data state version.
         *
         * @param toVersion the version
         */
        public Blob(long toVersion) {
            this(HollowConstants.VERSION_NONE, toVersion);
        }

        /**
         * Instantiate a delta from one data state version to another.
         *
         * @param fromVersion the version to start the delta from
         * @param toVersion the version to end the delta from
         */
        public Blob(long fromVersion, long toVersion) {
            this.fromVersion = fromVersion;
            this.toVersion = toVersion;

            if (this.isSnapshot())
                this.blobType = BlobType.SNAPSHOT;
            else if (this.isReverseDelta())
                this.blobType = BlobType.REVERSE_DELTA;
            else
                this.blobType = BlobType.DELTA;
        }

        /**
         * Implementations will define how to retrieve the actual blob data for this specific transition from a data store.
         *

         * It is expected that the returned InputStream will not be interrupted.  For this reason, it is a good idea to
         * retrieve the entire blob (e.g. to disk) from a remote datastore prior to returning this stream.
         *
         * @return the input stream to the blob
         * @throws IOException if the input stream to the blob cannot be obtained
         */
        public abstract InputStream getInputStream() throws IOException;

        /**
         * Blobs can be of types {@code SNAPSHOT}, {@code DELTA} or {@code REVERSE_DELTA}.
         */
        public enum BlobType {
            SNAPSHOT("snapshot"),
            DELTA("delta"),
            REVERSE_DELTA("reversedelta");

            private final String type;
            BlobType(String type) {
                this.type = type;
            }

            public String getType() {
                return this.type;
            }
        }

        public boolean isSnapshot() {
            return fromVersion == HollowConstants.VERSION_NONE;
        }

        public boolean isReverseDelta() {
            return toVersion < fromVersion;
        }

        public boolean isDelta() {
            return !isSnapshot() && !isReverseDelta();
        }

        public long getFromVersion() {
            return fromVersion;
        }

        public long getToVersion() {
            return toVersion;
        }

        public BlobType getBlobType() {
            return blobType;
        }
    }

    /**
     * Implementations of this class are responsible for two things:
     *

     * 1) Tracking the latest announced data state version.
     * 2) Keeping the client up to date by calling triggerAsyncRefresh() on self when the latest version changes.
     *

     * If an AnnouncementWatcher is provided to a HollowConsumer, then calling HollowConsumer#triggerRefreshTo() is unsupported.
     */
    public interface AnnouncementWatcher {

        long NO_ANNOUNCEMENT_AVAILABLE = HollowConstants.VERSION_NONE;

        /**
         * @return the latest announced version.
         */
        long getLatestVersion();

        /**
         * Implementations of this method should subscribe a HollowConsumer to updates to announced versions.
         *

         * When announcements are received via a push mechanism, or polling reveals a new version, a call should be placed to one
         * of the flavors of {@link HollowConsumer#triggerRefresh()} on the provided HollowConsumer.
         *
         * @param consumer the hollow consumer
         */
        void subscribeToUpdates(HollowConsumer consumer);
    }

    public interface DoubleSnapshotConfig {

        boolean allowDoubleSnapshot();

        int maxDeltasBeforeDoubleSnapshot();

        DoubleSnapshotConfig DEFAULT_CONFIG = new DoubleSnapshotConfig() {
            @Override
            public int maxDeltasBeforeDoubleSnapshot() {
                return 32;
            }

            @Override
            public boolean allowDoubleSnapshot() {
                return true;
            }
        };
    }

    public interface ObjectLongevityConfig {

        /**
         * @return whether or not long-lived object support is enabled.
         *

         * Because Hollow reuses pooled memory, if references to Hollow records are held too long, the underlying data may
         * be overwritten.  When long-lived object support is enabled, Hollow records referenced via a {@link HollowAPI} will,
         * after an update, be backed by a reserved copy of the data at the time the reference was created.  This guarantees
         * that even if a reference is held for a long time, it will continue to return the same data when interrogated.
         *

         * These reserved copies are backed by the {@link HollowHistory} data structure.
         */
        boolean enableLongLivedObjectSupport();

        boolean enableExpiredUsageStackTraces();

        /**
         * @return if long-lived object support is enabled, the number of milliseconds before the {@link StaleHollowReferenceDetector}
         * will begin flagging usage of stale objects.
         */
        long gracePeriodMillis();

        /**
         * @return if long-lived object support is enabled, the number of milliseconds, after the grace period, during which
         * data is still available in stale references, but usage will be flagged by the {@link StaleHollowReferenceDetector}.
         *

         * After the grace period + usage detection period have expired, the data from stale references will become inaccessible if
         * dropDataAutomatically() is enabled.
         */
        long usageDetectionPeriodMillis();

        /**
         * @return whether or not to drop data behind stale references after the grace period + usage detection period has elapsed, assuming
         * that no usage was detected during the usage detection period.
         */
        boolean dropDataAutomatically();

        /**
         * @return whether data is dropped even if flagged during the usage detection period.
         */
        boolean forceDropData();

        ObjectLongevityConfig DEFAULT_CONFIG = new ObjectLongevityConfig() {
            @Override
            public boolean enableLongLivedObjectSupport() {
                return false;
            }

            @Override
            public boolean dropDataAutomatically() {
                return false;
            }

            @Override
            public boolean forceDropData() {
                return false;
            }

            @Override
            public boolean enableExpiredUsageStackTraces() {
                return false;
            }

            @Override
            public long usageDetectionPeriodMillis() {
                return 60 * 60 * 1000;
            }

            @Override
            public long gracePeriodMillis() {
                return 60 * 60 * 1000;
            }
        };
    }

    /**
     * Listens for stale Hollow object usage
     */
    public interface ObjectLongevityDetector {

        /**
         * Stale reference detection hint.  This will be called every ~30 seconds.
         *

         * If a nonzero value is reported, then stale references to Hollow objects may be cached somewhere in your codebase.
         *

         * This signal can be noisy, and a nonzero value indicates that some reference to stale data exists somewhere.
         *
         * @param count the count of stale references
         */
        void staleReferenceExistenceDetected(int count);

        /**
         * Stale reference USAGE detection.  This will be called every ~30 seconds.
         *

         * If a nonzero value is reported, then stale references to Hollow objects are being accessed from somewhere in your codebase.
         *

         * This signal is noiseless, and a nonzero value indicates that some reference to stale data is USED somewhere.
         *
         * @param count the count of stale references
         */
        void staleReferenceUsageDetected(int count);

        ObjectLongevityDetector DEFAULT_DETECTOR = new ObjectLongevityDetector() {
            @Override
            public void staleReferenceUsageDetected(int count) {
            }

            @Override
            public void staleReferenceExistenceDetected(int count) {
            }
        };
    }

    /**
     * Implementations of this class will define what to do when various events happen before, during, and after updating
     * local in-memory copies of hollow data sets.
     */
    public interface RefreshListener {

        /**
         * Indicates that a refresh has begun.  Generally useful for logging.
         *

         * A refresh is the process of a consumer getting from a current version to a desired version.
         *

         * A refresh will consist of one of the following:
         *

         *
one or more deltas

         *
a snapshot load, plus zero or more deltas

         *

         *
         * @param currentVersion   the current state version
         * @param requestedVersion the version to which the refresh is progressing
         */
        void refreshStarted(long currentVersion, long requestedVersion);

        /**
         * This method is called when either data was initialized for the first time, or an update occurred across a
         * discontinuous delta chain (double snapshot).
         *

         * If this method is called, it means that the current refresh consists of a snapshot load, plus zero or more deltas.
         *

         * Implementations may initialize (or re-initialize) any indexing which is critical to keep in-sync with the data.
         *

         * This method will be called a maximum of once per refresh, after the data has reached the final state of the refresh.
         *
         * @param api         the {@link HollowAPI} instance
         * @param stateEngine the {@link HollowReadStateEngine}
         * @param version     the current state version
         * @throws Exception thrown if an error occurs in processing
         */
        void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;

        /**
         * This method is called whenever a live state engine's data is updated with a delta.  This method is not
         * called during first time initialization or when an update across a discontinuous delta chain (double snapshot)
         * occurs.
         *

         * Implementations should incrementally update any indexing which is critical to keep in-sync with the data.
         *

         * If this method is called, it means that the current refresh consists of one or more deltas, and does not include
         * a snapshot load.
         *

         * This method may be called multiple times per refresh, once for each time a delta is applied.
         *
         * @param api         the {@link HollowAPI} instance
         * @param stateEngine the {@link HollowReadStateEngine}
         * @param version     the current state version
         * @throws Exception thrown if an error occurs in processing
         */
        void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;

        /**
         * Called to indicate a blob was loaded (either a snapshot or delta).  Generally useful for logging or tracing of applied updates.
         *
         * @param transition The transition which was applied.
         */
        void blobLoaded(HollowConsumer.Blob transition);

        /**
         * Indicates that a refresh completed successfully.
         *
         * @param beforeVersion    - The version when the refresh started
         * @param afterVersion     - The version when the refresh completed
         * @param requestedVersion - The specific version which was requested
         */
        void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion);

        /**
         * Indicates that a refresh failed with an Exception.
         *
         * @param beforeVersion    - The version when the refresh started
         * @param afterVersion     - The version when the refresh completed
         * @param requestedVersion - The specific version which was requested
         * @param failureCause     - The Exception which caused the failure.
         */
        void refreshFailed(long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause);

    }

    public interface TransitionAwareRefreshListener extends RefreshListener {

        /**
         * This method is called whenever a snapshot is processed.  In the case of first time initialization or an update
         * across a discontinuous delta chain (double snapshot), this method will be called once (as the first transition).
         *

         * Implementations may initialize (or re-initialize) any indexing which is critical to keep in-sync with the data.
         *
         * @param api         the {@link HollowAPI} instance
         * @param stateEngine the {@link HollowReadStateEngine}
         * @param version     the current state version
         * @throws Exception thrown if an error occurs in processing
         */
        void snapshotApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;

        /**
         * This method is called whenever a delta is processed.  In the case of first time initialization or an update
         * across a discontinuous delta chain (double snapshot), this method may be called one or more times before arriving
         * at the final state (after which {@link #snapshotUpdateOccurred(HollowAPI, HollowReadStateEngine, long)} is called.
         *

         * Implementations may incrementally update any indexing which is critical to keep in-sync with the data.
         *
         * @param api         the {@link HollowAPI} instance
         * @param stateEngine the {@link HollowReadStateEngine}
         * @param version     the current state version
         * @throws Exception thrown if an error occurs in processing
         */
        void deltaApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;

        /**
         * Called after refresh started and update plan has been initialized, but before the update plan starts executing.
         * It is called only once per update plan (and thus only once per consumer refresh). Exposes details of the
         * update plan.
         * @implSpec The default implementation provided does nothing.
         *
         * @param beforeVersion The version when refresh started
         * @param desiredVersion The version that the consumer refresh tries update to, even though it might not be attainable eg. HollowConstants.VERSION_LATEST
         * @param isSnapshotPlan Indicates whether the refresh involves a snapshot transition
         * @param transitionSequence List of transitions comprising the refresh
         */
        default void transitionsPlanned(long beforeVersion, long desiredVersion, boolean isSnapshotPlan, List<HollowConsumer.Blob.BlobType> transitionSequence) {}
    }

    /**
     * A listener of refresh listener addition and removal.
     *

     * A {@link RefreshListener} implementation may  implement this interface to get notified before
     * the listener is added (via a call to {@link #addRefreshListener(RefreshListener)} and after a listener
     * is removed (via a call to {@link #removeRefreshListener(RefreshListener)}.
     *

     * An implementation should not add or remove itself in response to addition or removal.  Such actions may result
     * in a {@link StackOverflowError} or unspecified behaviour.
     */
    public interface RefreshRegistrationListener {
        /**
         * Called before the refresh listener is added.
         * @param c the consumer the associated reference listener is being added to
         */
        void onBeforeAddition(HollowConsumer c);

        /**
         * Called after the refresh listener is removed.
         * @param c the consumer the associated reference listener is being removed from
         */
        void onAfterRemoval(HollowConsumer c);
    }

    public static class AbstractRefreshListener implements TransitionAwareRefreshListener {
        @Override
        public void refreshStarted(long currentVersion, long requestedVersion) {
            // no-op
        }

        @Override
        public void transitionsPlanned(long beforeVersion, long desiredVersion, boolean isSnapshotPlan, List<HollowConsumer.Blob.BlobType> transitionSequence) {
            // no-op
        }

        @Override
        public void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
            // no-op
        }

        @Override
        public void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
            // no-op
        }

        @Override
        public void blobLoaded(Blob transition) {
            // no-op
        }

        @Override
        public void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion) {
            // no-op
        }

        @Override
        public void refreshFailed(long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause) {
            // no-op
        }

        @Override
        public void snapshotApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
            // no-op
        }

        @Override
        public void deltaApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
            // no-op
        }
    }

    public static HollowConsumer.Builder<?> withBlobRetriever(HollowConsumer.BlobRetriever blobRetriever) {
        HollowConsumer.Builder<?> builder = new Builder<>();
        return builder.withBlobRetriever(blobRetriever);
    }

    public static HollowConsumer.Builder<?> withLocalBlobStore(File localBlobStoreDir) {
        HollowConsumer.Builder<?> builder = new Builder<>();
        return builder.withLocalBlobStore(localBlobStoreDir);
    }

    @SuppressWarnings("unchecked")
    public static class Builder<B extends HollowConsumer.Builder> {

        protected HollowConsumer.BlobRetriever blobRetriever = null;
        protected HollowConsumer.AnnouncementWatcher announcementWatcher = null;
        protected HollowFilterConfig filterConfig = null;
        protected List<HollowConsumer.RefreshListener> refreshListeners = new ArrayList<>();
        protected HollowAPIFactory apiFactory = HollowAPIFactory.DEFAULT_FACTORY;
        protected HollowObjectHashCodeFinder hashCodeFinder = new DefaultHashCodeFinder();
        protected HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig = DoubleSnapshotConfig.DEFAULT_CONFIG;
        protected HollowConsumer.ObjectLongevityConfig objectLongevityConfig = ObjectLongevityConfig.DEFAULT_CONFIG;
        protected HollowConsumer.ObjectLongevityDetector objectLongevityDetector = ObjectLongevityDetector.DEFAULT_DETECTOR;
        protected File localBlobStoreDir = null;
        protected Executor refreshExecutor = null;
        protected HollowMetricsCollector<HollowConsumerMetrics> metricsCollector;

        public B withBlobRetriever(HollowConsumer.BlobRetriever blobRetriever) {
            this.blobRetriever = blobRetriever;
            return (B)this;
        }

        public B withLocalBlobStore(File localBlobStoreDir) {
            this.localBlobStoreDir = localBlobStoreDir;
            return (B)this;
        }

        public B withAnnouncementWatcher(HollowConsumer.AnnouncementWatcher announcementWatcher) {
            this.announcementWatcher = announcementWatcher;
            return (B)this;
        }

        public B withRefreshListener(HollowConsumer.RefreshListener refreshListener) {
            refreshListeners.add(refreshListener);
            return (B)this;
        }

        public B withRefreshListeners(HollowConsumer.RefreshListener... refreshListeners) {
            Collections.addAll(this.refreshListeners, refreshListeners);
            return (B)this;
        }

        /**
         * Provide the code generated API class that extends {@link HollowAPI}.
         *
         * The instance returned from {@link HollowConsumer#getAPI()} will be of the provided type and can be cast
         * to access generated methods.
         *
         * @param generatedAPIClass the code generated API class
         * @return this builder
         * @throws IllegalArgumentException if provided API class is {@code HollowAPI} instead of a subclass
         */
        public B withGeneratedAPIClass(Class<? extends HollowAPI> generatedAPIClass) {
            if (HollowAPI.class.equals(generatedAPIClass))
                throw new IllegalArgumentException("must provide a code generated API class");
            this.apiFactory = new HollowAPIFactory.ForGeneratedAPI<>(generatedAPIClass);
            return (B)this;
        }

        public B withFilterConfig(HollowFilterConfig filterConfig) {
            this.filterConfig = filterConfig;
            return (B)this;
        }

        public B withDoubleSnapshotConfig(HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig) {
            this.doubleSnapshotConfig = doubleSnapshotConfig;
            return (B)this;
        }

        public B withObjectLongevityConfig(HollowConsumer.ObjectLongevityConfig objectLongevityConfig) {
            this.objectLongevityConfig = objectLongevityConfig;
            return (B)this;
        }

        public B withObjectLongevityDetector(HollowConsumer.ObjectLongevityDetector objectLongevityDetector) {
            this.objectLongevityDetector = objectLongevityDetector;
            return (B)this;
        }

        public B withRefreshExecutor(Executor refreshExecutor) {
            this.refreshExecutor = refreshExecutor;
            return (B)this;
        }

        public B withMetricsCollector(HollowMetricsCollector<HollowConsumerMetrics> metricsCollector) {
            this.metricsCollector = metricsCollector;
            return (B)this;
        }

        @Deprecated
        public B withHashCodeFinder(HollowObjectHashCodeFinder hashCodeFinder) {
            this.hashCodeFinder = hashCodeFinder;
            return (B)this;
        }

        protected void checkArguments() {

            if (blobRetriever == null && localBlobStoreDir == null)
                throw new IllegalArgumentException("A HollowBlobRetriever or local blob store directory must be specified when building a HollowClient");

            BlobRetriever blobRetriever = this.blobRetriever;
            if (localBlobStoreDir != null)
                this.blobRetriever = new HollowFilesystemBlobRetriever(localBlobStoreDir.toPath(), blobRetriever);

            if (refreshExecutor == null)
                refreshExecutor = newSingleThreadExecutor(r -> daemonThread(r, getClass(), "refresh"));
        }

        public HollowConsumer build() {
            checkArguments();
            return new HollowConsumer(blobRetriever,
                    announcementWatcher,
                    refreshListeners,
                    apiFactory,
                    filterConfig,
                    objectLongevityConfig,
                    objectLongevityDetector,
                    doubleSnapshotConfig,
                    hashCodeFinder,
                    refreshExecutor,
                    metricsCollector);
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.util;

import com.netflix.hollow.core.read.engine.HollowBlobReader;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.write.HollowBlobWriter;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;

/**
 * A utility to create and keep up-to-date a {@link HollowReadStateEngine} from a {@link HollowWriteStateEngine}
 */
public class StateEngineRoundTripper {

    /**
     * @param writeEngine the write state engine
     * @return a brand-new {@link HollowReadStateEngine} with the dataset populated in the provided {@link HollowWriteStateEngine}
     * @throws IOException if the round trip from write to read state failed
     */
    public static HollowReadStateEngine roundTripSnapshot(HollowWriteStateEngine writeEngine) throws IOException {
        HollowReadStateEngine readEngine = new HollowReadStateEngine();
        roundTripSnapshot(writeEngine, readEngine);
        return readEngine;
    }

    /**
     * Populate the provided {@link HollowReadStateEngine} with the dataset currently in the provided {@link HollowWriteStateEngine}
     *
     * @param writeEngine the write state engine
     * @param readEngine the read state engine
     * @throws IOException if the round trip from write to read state failed
     */
    public static void roundTripSnapshot(HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine) throws IOException {
        roundTripSnapshot(writeEngine, readEngine, null);
    }

    /**
     * Populate the provided {@link HollowReadStateEngine} with the dataset currently in the provided {@link HollowWriteStateEngine}.
     *

     * Apply the provided {@link HollowFilterConfig}.
     *
     * @param writeEngine the write state engine
     * @param readEngine the read state engine
     * @param filter the filter configuration
     * @throws IOException if the round trip from write to read state failed
     */
    public static void roundTripSnapshot(HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine, HollowFilterConfig filter) throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        HollowBlobWriter writer = new HollowBlobWriter(writeEngine);
        writer.writeSnapshot(baos);
        writeEngine.prepareForNextCycle();

        HollowBlobReader reader = new HollowBlobReader(readEngine);
        InputStream is = new ByteArrayInputStream(baos.toByteArray());
        if(filter == null)
            reader.readSnapshot(is);
        else
            reader.readSnapshot(is, filter);
    }

    /**
     * Update the provided {@link HollowReadStateEngine} with the new state currently available in the {@link HollowWriteStateEngine}.
     *

     * It is assumed that the readEngine is currently populated with the prior state from the writeEngine.
     *
     * @param writeEngine the write state engine
     * @param readEngine the read state engine
     * @throws IOException if the round trip from write to read state failed
     */
    public static void roundTripDelta(HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine) throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        HollowBlobWriter writer = new HollowBlobWriter(writeEngine);
        writer.writeDelta(baos);
        HollowBlobReader reader = new HollowBlobReader(readEngine);
        reader.applyDelta(new ByteArrayInputStream(baos.toByteArray()));
        writeEngine.prepareForNextCycle();
    }

}