/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory;

import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import sun.misc.Unsafe;

/**
 * A segmented long array can grow without allocating successively larger blocks and copying memory.

 *
 * Segment length is always a power of two so that the location of a given index can be found with mask and shift operations.

 *
 * Conceptually this can be thought of as a single long array of undefined length.  The currently allocated buffer will always be
 * a multiple of the size of the segments.  The buffer will grow automatically when a byte is written to an index greater than the
 * currently allocated buffer.
 *
 * @author dkoszewnik
 *
 */
@SuppressWarnings("restriction")
public class SegmentedLongArray {

    private static final Unsafe unsafe = HollowUnsafeHandle.getUnsafe();

    protected final long[][] segments;
    protected final int log2OfSegmentSize;
    protected final int bitmask;

    public SegmentedLongArray(ArraySegmentRecycler memoryRecycler, long numLongs) {
        this.log2OfSegmentSize = memoryRecycler.getLog2OfLongSegmentSize();
        int numSegments = (int)((numLongs - 1) >>> log2OfSegmentSize) + 1;
        long[][] segments = new long[numSegments][];
        this.bitmask = (1 << log2OfSegmentSize) - 1;

        for(int i=0;i<segments.length;i++) {
            segments[i] = memoryRecycler.getLongArray();
        }

        /// The following assignment is purposefully placed *after* the population of all segments.
        /// The final assignment after the initialization of the array guarantees that no thread
        /// will see any of the array elements before assignment.
        /// We can't risk the segment values being visible as null to any thread, because
        /// FixedLengthElementArray uses Unsafe to access these values, which would cause the
        /// JVM to crash with a segmentation fault.
        this.segments = segments;
    }

    /**
     * Set the byte at the given index to the specified value
     *
     * @param index the index
     * @param value the byte value
     */
    public void set(long index, long value) {
        int segmentIndex = (int)(index >> log2OfSegmentSize);
        int longInSegment = (int)(index & bitmask);
        unsafe.putOrderedLong(segments[segmentIndex], (long)Unsafe.ARRAY_LONG_BASE_OFFSET + (8 * longInSegment), value);

        /// duplicate the longs here so that we can read faster.
        if(longInSegment == 0 && segmentIndex != 0)
            unsafe.putOrderedLong(segments[segmentIndex - 1], (long)Unsafe.ARRAY_LONG_BASE_OFFSET + (8 * (1 << log2OfSegmentSize)), value);
    }

    /**
     * Get the value of the byte at the specified index.
     *
     * @param index the index
     * @return the byte value
     */
    public long get(long index) {
        int segmentIndex = (int)(index >>> log2OfSegmentSize);
        return segments[segmentIndex][(int)(index & bitmask)];
    }

    public void fill(long value) {
        for(int i=0;i<segments.length;i++) {
            long offset = Unsafe.ARRAY_LONG_BASE_OFFSET;
            for(int j=0;j<segments[i].length;j++) {
                unsafe.putOrderedLong(segments[i], offset, value);
                offset += 8;
            }
        }
    }

    public void writeTo(DataOutputStream dos, long numLongs) throws IOException {
        VarInt.writeVLong(dos, numLongs);

        for(long i=0;i<numLongs;i++) {
            dos.writeLong(get(i));
        }
    }

    public void destroy(ArraySegmentRecycler memoryRecycler) {
        for(int i=0;i<segments.length;i++) {
            if(segments[i] != null)
                memoryRecycler.recycleLongArray(segments[i]);
        }
    }

    public static SegmentedLongArray deserializeFrom(DataInputStream dis, ArraySegmentRecycler memoryRecycler) throws IOException {
        long numLongs = VarInt.readVLong(dis);

        SegmentedLongArray arr = new SegmentedLongArray(memoryRecycler, numLongs);

        arr.readFrom(dis, memoryRecycler, numLongs);

        return arr;
    }

    protected void readFrom(DataInputStream dis, ArraySegmentRecycler memoryRecycler, long numLongs) throws IOException {
        int segmentSize = 1 << memoryRecycler.getLog2OfLongSegmentSize();
        int segment = 0;

        if(numLongs == 0)
            return;

        long fencepostLong = dis.readLong();

        while(numLongs > 0) {
            long longsToCopy = Math.min(segmentSize, numLongs);

            unsafe.putOrderedLong(segments[segment], (long)Unsafe.ARRAY_LONG_BASE_OFFSET, fencepostLong);

            int longsCopied = 1;

            while(longsCopied < longsToCopy) {
                long l = dis.readLong();
                unsafe.putOrderedLong(segments[segment], (long)Unsafe.ARRAY_LONG_BASE_OFFSET + (8 * longsCopied++), l);
            }

            if(numLongs > longsCopied) {
                unsafe.putOrderedLong(segments[segment], (long)Unsafe.ARRAY_LONG_BASE_OFFSET + (8 * longsCopied), dis.readLong());
                fencepostLong = segments[segment][longsCopied];
            }

            segment++;
            numLongs -= longsCopied;
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.producer.fs;

import com.netflix.hollow.api.producer.HollowProducer;
import java.io.File;
import java.io.FilenameFilter;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.logging.Logger;

public class HollowFilesystemBlobStorageCleaner extends HollowProducer.BlobStorageCleaner {

    private final Logger log = Logger.getLogger(HollowFilesystemBlobStorageCleaner.class.getName());

    private int numOfSnapshotsToKeep;
    private final File blobStoreDir;

    public HollowFilesystemBlobStorageCleaner(File blobStoreDir) {
        this(blobStoreDir,5);
    }

    public HollowFilesystemBlobStorageCleaner(File blobStoreDir, int numOfSnapshotsToKeep) {
        this.blobStoreDir = blobStoreDir;
        this.numOfSnapshotsToKeep = numOfSnapshotsToKeep;
    }

    /**
     * Cleans snapshot to keep the last 'n' snapshots. Defaults to 5.
     */
    @Override
    public void cleanSnapshots() {
        File[] files = getFilesByType(HollowProducer.Blob.Type.SNAPSHOT.prefix);

        if(files == null || files.length <= numOfSnapshotsToKeep) {
            return;
        }

        sortByLastModified(files);

        for(int i= numOfSnapshotsToKeep; i < files.length; i++){
            File file = files[i];
            boolean deleted = file.delete();
            if(!deleted) {
                log.warning("Could not delete snapshot " + file.getPath());
            }
        }
    }

    @Override
    public void cleanDeltas() { }

    @Override
    public void cleanReverseDeltas() { }

    private void sortByLastModified(File[] files) {
        Arrays.sort(files, new Comparator<File>() {
            public int compare(File f1, File f2) {
                Long lastModifiedF2 = f2.lastModified();
                Long lastModifiedF1 = f1.lastModified();
                return lastModifiedF2.compareTo(lastModifiedF1);
            }
        });
        Arrays.sort(files, Collections.reverseOrder());
    }

    private File[] getFilesByType(final String blobType) {
        return blobStoreDir.listFiles(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                return name.contains(blobType);
            }
        });
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.read.dataaccess.HollowDataAccess;
import com.netflix.hollow.core.read.dataaccess.HollowObjectTypeDataAccess;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import java.util.Arrays;

/**
 * Useful utility methods for interacting with a {@link HollowDataAccess}.
 */
public class HollowReadFieldUtils {

    /**
     * Hash a field in an OBJECT record.
     *
     * @param typeAccess the data access
     * @param ordinal the ordinal
     * @param fieldPosition the field position
     * @return the hash code
     */
    public static int fieldHashCode(HollowObjectTypeDataAccess typeAccess, int ordinal, int fieldPosition) {
        HollowObjectSchema schema = typeAccess.getSchema();

        switch(schema.getFieldType(fieldPosition)) {
            case BOOLEAN:
                Boolean bool = typeAccess.readBoolean(ordinal, fieldPosition);
                return booleanHashCode(bool);
            case BYTES:
            case STRING:
                return typeAccess.findVarLengthFieldHashCode(ordinal, fieldPosition);
            case DOUBLE:
                double d = typeAccess.readDouble(ordinal, fieldPosition);
                return doubleHashCode(d);
            case FLOAT:
                float f = typeAccess.readFloat(ordinal, fieldPosition);
                return floatHashCode(f);
            case INT:
                return intHashCode(typeAccess.readInt(ordinal, fieldPosition));
            case LONG:
                long l = typeAccess.readLong(ordinal, fieldPosition);
                return longHashCode(l);
            case REFERENCE:
                return typeAccess.readOrdinal(ordinal, fieldPosition);
        }

        throw new IllegalStateException("I don't know how to hash a " + schema.getFieldType(fieldPosition));
    }

    /**
     * Determine whether two OBJECT field records are exactly equal.
     *
     * @param typeAccess1 the first type access
     * @param ordinal1 the first ordinal
     * @param fieldPosition1 the first field positiom
     * @param typeAccess2 the second type access
     * @param ordinal2 the second ordinal
     * @param fieldPosition2 the second field position
     *
     * @return if the two OBJECT field records are exactly equal
     */
    public static boolean fieldsAreEqual(HollowObjectTypeDataAccess typeAccess1, int ordinal1, int fieldPosition1, HollowObjectTypeDataAccess typeAccess2, int ordinal2, int fieldPosition2) {
        HollowObjectSchema schema1 = typeAccess1.getSchema();

        switch(schema1.getFieldType(fieldPosition1)) {
            case BOOLEAN:
                Boolean bool1 = typeAccess1.readBoolean(ordinal1, fieldPosition1);
                Boolean bool2 = typeAccess2.readBoolean(ordinal2, fieldPosition2);
                return bool1 == bool2;
            case BYTES:
                byte[] data1 = typeAccess1.readBytes(ordinal1, fieldPosition1);
                byte[] data2 = typeAccess2.readBytes(ordinal2, fieldPosition2);
                return Arrays.equals(data1, data2);
            case DOUBLE:
                double d1 = typeAccess1.readDouble(ordinal1, fieldPosition1);
                double d2 = typeAccess2.readDouble(ordinal2, fieldPosition2);
                return Double.compare(d1, d2) == 0;
            case FLOAT:
                float f1 = typeAccess1.readFloat(ordinal1, fieldPosition1);
                float f2 = typeAccess2.readFloat(ordinal2, fieldPosition2);
                return Float.compare(f1, f2) == 0;
            case INT:
                int i1 = typeAccess1.readInt(ordinal1, fieldPosition1);
                int i2 = typeAccess2.readInt(ordinal2, fieldPosition2);
                return i1 == i2;
            case LONG:
                long l1 = typeAccess1.readLong(ordinal1, fieldPosition1);
                long l2 = typeAccess2.readLong(ordinal2, fieldPosition2);
                return l1 == l2;
            case STRING:
                String s1 = typeAccess1.readString(ordinal1, fieldPosition1);
                return typeAccess2.isStringFieldEqual(ordinal2, fieldPosition2, s1);
            case REFERENCE:
                if(typeAccess1 == typeAccess2 && fieldPosition1 == fieldPosition2)
                    return typeAccess1.readOrdinal(ordinal1, fieldPosition1) == typeAccess2.readOrdinal(ordinal2, fieldPosition2);
            default:
        }

        throw new IllegalStateException("I don't know how to test equality for a " + schema1.getFieldType(fieldPosition1));
    }

    /**
     * @param typeAccess the type access
     * @param ordinal the ordinal
     * @param fieldPosition the field position
     * @return a displayable String for a field from an OBJECT record.
     */
    public static String displayString(HollowObjectTypeDataAccess typeAccess, int ordinal, int fieldPosition) {
        HollowObjectSchema schema = typeAccess.getSchema();

        switch(schema.getFieldType(fieldPosition)) {
            case BOOLEAN:
                Boolean bool = typeAccess.readBoolean(ordinal, fieldPosition);
                String.valueOf(bool);
            case BYTES:
            case STRING:
                return typeAccess.readString(ordinal, fieldPosition);
            case DOUBLE:
                double d = typeAccess.readDouble(ordinal, fieldPosition);
                return String.valueOf(d);
            case FLOAT:
                float f = typeAccess.readFloat(ordinal, fieldPosition);
                return String.valueOf(f);
            case INT:
                return String.valueOf(typeAccess.readInt(ordinal, fieldPosition));
            case LONG:
                long l = typeAccess.readLong(ordinal, fieldPosition);
                return String.valueOf(l);
            default:
        }

        throw new IllegalStateException("I don't know how to display a " + schema.getFieldType(fieldPosition));
    }

    /**
     * @param typeAccess the type access
     * @param ordinal the ordinal
     * @param fieldPosition the field position
     * @return an appropriate Object representing a Hollow OBJECT record field's value
     */
    public static Object fieldValueObject(HollowObjectTypeDataAccess typeAccess, int ordinal, int fieldPosition) {
        HollowObjectSchema schema = typeAccess.getSchema();

        switch(schema.getFieldType(fieldPosition)) {
            case BOOLEAN:
                return typeAccess.readBoolean(ordinal, fieldPosition);
            case BYTES:
                return typeAccess.readBytes(ordinal, fieldPosition);
            case STRING:
                return typeAccess.readString(ordinal, fieldPosition);
            case DOUBLE:
                double d = typeAccess.readDouble(ordinal, fieldPosition);
                return Double.isNaN(d) ? null : Double.valueOf(d);
            case FLOAT:
                float f = typeAccess.readFloat(ordinal, fieldPosition);
                return Float.isNaN(f) ? null : Float.valueOf(f);
            case INT:
                int i = typeAccess.readInt(ordinal, fieldPosition);
                if(i == Integer.MIN_VALUE)
                    return null;
                return Integer.valueOf(i);
            case LONG:
                long l = typeAccess.readLong(ordinal, fieldPosition);
                if(l == Long.MIN_VALUE)
                    return null;
                return Long.valueOf(l);
            case REFERENCE:
                int refOrdinal = typeAccess.readOrdinal(ordinal, fieldPosition);
                if(refOrdinal < 0)
                    return null;
                return Integer.valueOf(refOrdinal);
            default:
        }

        throw new IllegalStateException("Can't handle fieldType=" + schema.getFieldType(fieldPosition) + " for schema=" + schema.getName() + ", fieldPosition=" + fieldPosition);
    }

    /**
     * @param typeAccess the type access
     * @param ordinal the ordinal
     * @param fieldPosition the field position
     * @param testObject the object to test
     * @return whether the provided Object matches a Hollow OBJECT record's value.
     */
    public static boolean fieldValueEquals(HollowObjectTypeDataAccess typeAccess, int ordinal, int fieldPosition, Object testObject) {
        HollowObjectSchema schema = typeAccess.getSchema();

        switch(schema.getFieldType(fieldPosition)) {
            case BOOLEAN:
                if(testObject instanceof Boolean)
                    return testObject.equals(typeAccess.readBoolean(ordinal, fieldPosition));
                return testObject == null && typeAccess.readBoolean(ordinal, fieldPosition) == null;
            case BYTES:
                if(testObject instanceof byte[])
                    return Arrays.equals(typeAccess.readBytes(ordinal, fieldPosition), (byte[])testObject);
                return testObject == null && typeAccess.readBytes(ordinal, fieldPosition) == null;
            case STRING:
                if(testObject instanceof String)
                    return testObject.equals(typeAccess.readString(ordinal, fieldPosition));
                return testObject == null && typeAccess.readString(ordinal, fieldPosition) == null;
            case DOUBLE:
                if(testObject instanceof Double)
                    return testObject.equals(typeAccess.readDouble(ordinal, fieldPosition));
                return testObject == null && !Double.isNaN(typeAccess.readDouble(ordinal, fieldPosition));
            case FLOAT:
                if(testObject instanceof Float)
                    return testObject.equals(typeAccess.readFloat(ordinal, fieldPosition));
                return testObject == null && !Float.isNaN(typeAccess.readFloat(ordinal, fieldPosition));
            case INT:
                if(testObject instanceof Integer)
                    return testObject.equals(typeAccess.readInt(ordinal, fieldPosition));
                return testObject == null && typeAccess.readInt(ordinal, fieldPosition) == Integer.MIN_VALUE;
            case LONG:
                if(testObject instanceof Long)
                    return testObject.equals(typeAccess.readLong(ordinal, fieldPosition));
                return testObject == null && typeAccess.readLong(ordinal, fieldPosition) == Long.MIN_VALUE;
            case REFERENCE:
                if(testObject instanceof Integer)
                    return testObject.equals(typeAccess.readOrdinal(ordinal, fieldPosition));
                return testObject == null && typeAccess.readOrdinal(ordinal, fieldPosition) < 0;
            default:
        }

        throw new IllegalStateException("I don't know how to test equality for a " + schema.getFieldType(fieldPosition));
    }

    /**
     * @param data the byte array
     * @return The hash code for a byte array which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int byteArrayHashCode(byte[] data) {
        return HashCodes.hashCode(data);
    }

    /**
     * @param str the string value
     * @return The hash code for a String which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int stringHashCode(String str) {
        return HashCodes.hashCode(str);
    }

    /**
     * @param bool the boolean value
     * @return The hash code for a boolean which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int booleanHashCode(Boolean bool) {
        return bool == null ? -1 : bool ? 1 : 0;
    }

    /**
     * @param l the long value
     * @return The hash code for a long which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int longHashCode(long l) {
        return (int)l ^ (int)(l >> 32);
    }

    /**
     * @param i the int value
     * @return The hash code for an int which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int intHashCode(int i) {
        return i;
    }

    /**
     * @param f the float value
     * @return The hash code for a float which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int floatHashCode(float f) {
        return Float.floatToIntBits(f);
    }

    /**
     * @param d the double value
     * @return The hash code for a double which would be returned from {@link #fieldHashCode(HollowObjectTypeDataAccess, int, int)}
     */
    public static int doubleHashCode(double d) {
        long bits = Double.doubleToLongBits(d);
        return longHashCode(bits);
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.tools.traverse;

import static com.netflix.hollow.tools.traverse.TransitiveSetTraverser.TransitiveSetTraverserAction.ADD_REFERENCING_OUTSIDE_CLOSURE;
import static com.netflix.hollow.tools.traverse.TransitiveSetTraverser.TransitiveSetTraverserAction.REMOVE_REFERENCED_OUTSIDE_CLOSURE;

import com.netflix.hollow.core.read.engine.HollowCollectionTypeReadState;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.map.HollowMapTypeReadState;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.iterator.HollowMapEntryOrdinalIterator;
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
import com.netflix.hollow.core.schema.HollowCollectionSchema;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchemaSorter;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * The TransitiveSetTraverser can be used to find children and parent references for a selected set of records.
 * A selection is represented with a Map<String, BitSet>, where each key in the map represents a type, and the corresponding BitSet
 * represents the ordinals of the selected records.
 * Entries in this Map will indicate a type, plus the ordinals of the selected records:
 *

 * {@code
 * Map<String, BitSet> selection = new HashMap<String, BitSet>();
 *
 * /// select the movies with ordinals 1 and 4.
 * BitSet selectedMovies = new BitSet();
 * selectedMovies.set(1);
 * selectedMovies.set(4);
 *
 * selection.put("Movie", movies);
 * }
 *

 *

 * We can add the references, and the transitive references, of our selection.
 * After the following call returns, our selection will be augmented with these matches:
 *

 * {@code TransitiveSetTraverser.addTransitiveMatches(readEngine, selection);}
 *

 *

 * Given a selection, we can also add any records which reference anything in the selection.
 * This is essentially the opposite operation as above; it can be said that addTransitiveMatches
 * traverses down, while addReferencingOutsideClosure traverses up.  After the following call returns,
 * our selection will be augmented with this selection:
 *
 */
public class TransitiveSetTraverser {
    private static final Logger log = Logger.getLogger(TransitiveSetTraverser.class.getName());

    /**
     * Augment the given selection by adding the references, and the transitive references, of our selection.
     * @param stateEngine the state engine
     * @param matches the map to which matches are placed
     */
    public static void addTransitiveMatches(HollowReadStateEngine stateEngine, Map<String, BitSet> matches) {
        List<HollowSchema> schemaList = HollowSchemaSorter.dependencyOrderedSchemaList(stateEngine);
        Collections.reverse(schemaList);

        for(HollowSchema schema : schemaList) {
            BitSet currentMatches = matches.get(schema.getName());
            if(currentMatches != null) {
                addTransitiveMatches(stateEngine, schema.getName(), matches);
            }
        }
    }

    /**
     * Remove any records from the given selection which are referenced by other records not in the selection.
     * @param stateEngine the state engine
     * @param matches the matches
     */
    public static void removeReferencedOutsideClosure(HollowReadStateEngine stateEngine, Map<String, BitSet> matches) {
        List<HollowSchema> orderedSchemas = HollowSchemaSorter.dependencyOrderedSchemaList(stateEngine);
        Collections.reverse(orderedSchemas);

        for(HollowSchema referencedSchema : orderedSchemas) {
            if(matches.containsKey(referencedSchema.getName())) {
                for(HollowSchema referencerSchema : orderedSchemas) {
                    if(referencerSchema == referencedSchema)
                        break;

                    if(matches.containsKey(referencedSchema.getName()) && matches.get(referencedSchema.getName()).cardinality() > 0)
                        traverseReferencesOutsideClosure(stateEngine, referencerSchema.getName(), referencedSchema.getName(), matches, REMOVE_REFERENCED_OUTSIDE_CLOSURE);
                }
            }
        }
    }

    /**
     * Augment the given selection with any records outside the selection which reference
     * (or transitively reference) any records in the selection.
     * @param stateEngine the state engine
     * @param matches the matches
     */
    public static void addReferencingOutsideClosure(HollowReadStateEngine stateEngine, Map<String, BitSet> matches) {
        List<HollowSchema> orderedSchemas = HollowSchemaSorter.dependencyOrderedSchemaList(stateEngine);

        for(HollowSchema referencerSchema : orderedSchemas) {
            for(HollowSchema referencedSchema : orderedSchemas) {
                if(referencedSchema == referencerSchema)
                    break;

                if(matches.containsKey(referencedSchema.getName()) && matches.get(referencedSchema.getName()).cardinality() > 0)
                    traverseReferencesOutsideClosure(stateEngine, referencerSchema.getName(), referencedSchema.getName(), matches, ADD_REFERENCING_OUTSIDE_CLOSURE);
            }
        }
    }

    private static void addTransitiveMatches(HollowReadStateEngine stateEngine, String type, Map<String, BitSet> matches) {
        HollowTypeReadState typeState = stateEngine.getTypeState(type);

        switch(typeState.getSchema().getSchemaType()) {
        case OBJECT:
            addTransitiveMatches(stateEngine, (HollowObjectTypeReadState)typeState, matches);
            break;
        case LIST:
        case SET:
            addTransitiveMatches(stateEngine, (HollowCollectionTypeReadState)typeState, matches);
            break;
        case MAP:
            addTransitiveMatches(stateEngine, (HollowMapTypeReadState)typeState, matches);
            break;
        }
    }

    private static void addTransitiveMatches(HollowReadStateEngine stateEngine, HollowObjectTypeReadState typeState, Map<String, BitSet> matches) {
        HollowObjectSchema schema = typeState.getSchema();
        BitSet matchingOrdinals = getOrCreateBitSet(matches, schema.getName(), typeState.maxOrdinal());

        BitSet childOrdinals[] = new BitSet[schema.numFields()];

        for(int i=0;i<schema.numFields();i++) {
            if(schema.getFieldType(i) == FieldType.REFERENCE) {
                HollowTypeReadState childTypeState = stateEngine.getTypeState(schema.getReferencedType(i));
                if(childTypeState != null && childTypeState.maxOrdinal() >= 0)
                    childOrdinals[i] = getOrCreateBitSet(matches, schema.getReferencedType(i), childTypeState.maxOrdinal());
            }
        }

        int ordinal = matchingOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            for(int i=0;i<childOrdinals.length;i++) {
                if(childOrdinals[i] != null) {
                    int childOrdinal = typeState.readOrdinal(ordinal, i);
                    if(childOrdinal != -1) {
                        childOrdinals[i].set(childOrdinal);
                    }
                }
            }
            ordinal = matchingOrdinals.nextSetBit(ordinal + 1);
        }
    }

    private static void addTransitiveMatches(HollowReadStateEngine stateEngine, HollowCollectionTypeReadState typeState, Map<String, BitSet> matches) {
        HollowCollectionSchema schema = typeState.getSchema();
        BitSet matchingOrdinals = getOrCreateBitSet(matches, schema.getName(), typeState.maxOrdinal());

        HollowTypeReadState childTypeState = stateEngine.getTypeState(schema.getElementType());
        BitSet childOrdinals = getOrCreateBitSet(matches, schema.getElementType(), childTypeState.maxOrdinal());

        int ordinal = matchingOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            try {
                HollowOrdinalIterator iter = typeState.ordinalIterator(ordinal);
                int elementOrdinal = iter.next();
                while(elementOrdinal != HollowOrdinalIterator.NO_MORE_ORDINALS) {
                    childOrdinals.set(elementOrdinal);
                    elementOrdinal = iter.next();
                }
            } catch(Exception e) {
                log.log(Level.SEVERE, "Add transitive matches failed", e);
            }

            ordinal = matchingOrdinals.nextSetBit(ordinal + 1);
        }

        if(!childOrdinals.isEmpty()) {
            matches.put(schema.getElementType(), childOrdinals);
        }
    }

    private static void addTransitiveMatches(HollowReadStateEngine stateEngine, HollowMapTypeReadState typeState, Map<String, BitSet> matches) {
        HollowMapSchema schema = typeState.getSchema();
        BitSet matchingOrdinals = getOrCreateBitSet(matches, schema.getName(), typeState.maxOrdinal());

        HollowTypeReadState keyTypeState = stateEngine.getTypeState(schema.getKeyType());
        HollowTypeReadState valueTypeState = stateEngine.getTypeState(schema.getValueType());

        BitSet keyOrdinals = getOrCreateBitSet(matches, schema.getKeyType(), keyTypeState.maxOrdinal());
        BitSet valueOrdinals = getOrCreateBitSet(matches, schema.getValueType(), valueTypeState.maxOrdinal());

        int ordinal = matchingOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            HollowMapEntryOrdinalIterator iter = typeState.ordinalIterator(ordinal);
            while(iter.next()) {
                keyOrdinals.set(iter.getKey());
                valueOrdinals.set(iter.getValue());
            }

            ordinal = matchingOrdinals.nextSetBit(ordinal + 1);
        }
    }

    private static void traverseReferencesOutsideClosure(HollowReadStateEngine stateEngine, String referencerType, String referencedType, Map<String, BitSet> matches, TransitiveSetTraverserAction action) {
        HollowTypeReadState referencerTypeState = stateEngine.getTypeState(referencerType);

        switch(referencerTypeState.getSchema().getSchemaType()) {
        case OBJECT:
            traverseReferencesOutsideClosure(stateEngine, (HollowObjectTypeReadState)referencerTypeState, referencedType, matches, action);
            break;
        case LIST:
        case SET:
            traverseReferencesOutsideClosure(stateEngine, (HollowCollectionTypeReadState)referencerTypeState, referencedType, matches, action);
            break;
        case MAP:
            traverseReferencesOutsideClosure(stateEngine, (HollowMapTypeReadState)referencerTypeState, referencedType, matches, action);
            break;
        }
    }

    private static void traverseReferencesOutsideClosure(HollowReadStateEngine stateEngine, HollowObjectTypeReadState referencerTypeState, String referencedType, Map<String, BitSet> closureMatches, TransitiveSetTraverserAction action) {
        HollowObjectSchema schema = referencerTypeState.getSchema();
        BitSet referencedClosureMatches = getOrCreateBitSet(closureMatches, referencedType, stateEngine.getTypeState(referencedType).maxOrdinal());
        BitSet referencerClosureMatches = getOrCreateBitSet(closureMatches, schema.getName(), referencerTypeState.maxOrdinal());

        for(int i=0;i<schema.numFields();i++) {
            if(schema.getFieldType(i) == FieldType.REFERENCE && referencedType.equals(schema.getReferencedType(i))) {
                BitSet allReferencerOrdinals = getPopulatedOrdinals(referencerTypeState);

                int ordinal = allReferencerOrdinals.nextSetBit(0);
                while(ordinal != -1) {
                    if(!referencerClosureMatches.get(ordinal)) {
                        int refOrdinal = referencerTypeState.readOrdinal(ordinal, i);
                        if(refOrdinal != -1) {
                            if(referencedClosureMatches.get(refOrdinal)) {
                                action.foundReference(referencerClosureMatches, ordinal, referencedClosureMatches, refOrdinal);
                            }
                        }
                    }

                    ordinal = allReferencerOrdinals.nextSetBit(ordinal + 1);
                }
            }
        }
    }

    private static void traverseReferencesOutsideClosure(HollowReadStateEngine stateEngine, HollowCollectionTypeReadState referencerTypeState, String referencedType, Map<String, BitSet> closureMatches, TransitiveSetTraverserAction action) {
        HollowCollectionSchema schema = referencerTypeState.getSchema();

        if(!referencedType.equals(schema.getElementType()))
            return;

        BitSet referencedClosureMatches = getOrCreateBitSet(closureMatches, referencedType, stateEngine.getTypeState(referencedType).maxOrdinal());
        BitSet referencerClosureMatches = getOrCreateBitSet(closureMatches, schema.getName(), referencerTypeState.maxOrdinal());

        BitSet allReferencerOrdinals = getPopulatedOrdinals(referencerTypeState);

        int ordinal = allReferencerOrdinals.nextSetBit(0);
        while(ordinal != -1) {
            if(!referencerClosureMatches.get(ordinal)) {
                HollowOrdinalIterator iter = referencerTypeState.ordinalIterator(ordinal);
                int refOrdinal = iter.next();
                while(refOrdinal != HollowOrdinalIterator.NO_MORE_ORDINALS) {
                    if(referencedClosureMatches.get(refOrdinal)) {
                        action.foundReference(referencerClosureMatches, ordinal, referencedClosureMatches, refOrdinal);
                    }
                    refOrdinal = iter.next();
                }
            }

            ordinal = allReferencerOrdinals.nextSetBit(ordinal + 1);
        }
    }

    private static void traverseReferencesOutsideClosure(HollowReadStateEngine stateEngine, HollowMapTypeReadState referencerTypeState, String referencedType, Map<String, BitSet> closureMatches, TransitiveSetTraverserAction action) {
        HollowMapSchema schema = referencerTypeState.getSchema();

        BitSet referencedClosureMatches = getOrCreateBitSet(closureMatches, referencedType, stateEngine.getTypeState(referencedType).maxOrdinal());
        BitSet referencerClosureMatches = getOrCreateBitSet(closureMatches, schema.getName(), referencerTypeState.maxOrdinal());

        BitSet allReferencerOrdinals = getPopulatedOrdinals(referencerTypeState);

        boolean keyTypeMatches = referencedType.equals(schema.getKeyType());
        boolean valueTypeMatches = referencedType.equals(schema.getValueType());

        if(keyTypeMatches || valueTypeMatches) {
            int ordinal = allReferencerOrdinals.nextSetBit(0);
            while(ordinal != -1) {
                if(!referencerClosureMatches.get(ordinal)) {
                    HollowMapEntryOrdinalIterator iter = referencerTypeState.ordinalIterator(ordinal);
                    while(iter.next()) {
                        if(keyTypeMatches) {
                            int refOrdinal = iter.getKey();
                            if(referencedClosureMatches.get(refOrdinal)) {
                                action.foundReference(referencerClosureMatches, ordinal, referencedClosureMatches, refOrdinal);
                            }
                        }
                        if(valueTypeMatches) {
                            int refOrdinal = iter.getValue();
                            if(referencedClosureMatches.get(refOrdinal)) {
                                action.foundReference(referencerClosureMatches, ordinal, referencedClosureMatches, refOrdinal);
                            }
                        }
                    }
                }

                ordinal = allReferencerOrdinals.nextSetBit(ordinal + 1);
            }
        }
    }

    private static BitSet getPopulatedOrdinals(HollowTypeReadState typeState) {
        return typeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals();
    }

    private static BitSet getOrCreateBitSet(Map<String, BitSet> bitSets, String typeName, int numBitsRequired) {
        if(numBitsRequired < 0)
            numBitsRequired = 0;

        BitSet bs = bitSets.get(typeName);
        if(bs == null) {
            bs = new BitSet(numBitsRequired);
            bitSets.put(typeName, bs);
        }
        return bs;
    }

    public static interface TransitiveSetTraverserAction {

        public void foundReference(BitSet referencerClosureMatches, int referencerOrdinal, BitSet referencedClosureMatches, int referencedOrdinal);

        public static final TransitiveSetTraverserAction REMOVE_REFERENCED_OUTSIDE_CLOSURE = new TransitiveSetTraverserAction() {
            @Override
            public void foundReference(BitSet referencerClosureMatches, int referencerOrdinal, BitSet referencedClosureMatches, int referencedOrdinal) {
                referencedClosureMatches.clear(referencedOrdinal);
            }
        };

        public static final TransitiveSetTraverserAction ADD_REFERENCING_OUTSIDE_CLOSURE = new TransitiveSetTraverserAction() {
            @Override
            public void foundReference(BitSet referencerClosureMatches, int referencerOrdinal, BitSet referencedClosureMatches, int referencedOrdinal) {
                referencerClosureMatches.set(referencerOrdinal);
            }
        };

    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.client;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.api.metrics.HollowConsumerMetrics;
import com.netflix.hollow.api.metrics.HollowMetricsCollector;
import com.netflix.hollow.core.HollowConstants;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.util.HollowObjectHashCodeFinder;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.logging.Logger;

/**
 * A class comprising much of the internal state of a {@link HollowConsumer}.  Not intended for external consumption.
 */
public class HollowClientUpdater {
    private static final Logger LOG = Logger.getLogger(HollowClientUpdater.class.getName());

    private volatile HollowDataHolder hollowDataHolderVolatile;

    private final HollowUpdatePlanner planner;
    private final CompletableFuture<Long> initialLoad;
    private boolean forceDoubleSnapshot = false;
    private final FailedTransitionTracker failedTransitionTracker;
    private final StaleHollowReferenceDetector staleReferenceDetector;

    private final CopyOnWriteArrayList<HollowConsumer.RefreshListener> refreshListeners;
    private final HollowAPIFactory apiFactory;
    private final HollowObjectHashCodeFinder hashCodeFinder;
    private final HollowConsumer.ObjectLongevityConfig objectLongevityConfig;
    private final HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig;
    private final HollowConsumerMetrics metrics;
    private final HollowMetricsCollector<HollowConsumerMetrics> metricsCollector;

    private HollowFilterConfig filter;

    public HollowClientUpdater(HollowConsumer.BlobRetriever transitionCreator,
                               List<HollowConsumer.RefreshListener> refreshListeners,
                               HollowAPIFactory apiFactory,
                               HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig,
                               HollowObjectHashCodeFinder hashCodeFinder,
                               HollowConsumer.ObjectLongevityConfig objectLongevityConfig,
                               HollowConsumer.ObjectLongevityDetector objectLongevityDetector,
                               HollowConsumerMetrics metrics,
                               HollowMetricsCollector<HollowConsumerMetrics> metricsCollector) {
        this.planner = new HollowUpdatePlanner(transitionCreator, doubleSnapshotConfig);
        this.failedTransitionTracker = new FailedTransitionTracker();
        this.staleReferenceDetector = new StaleHollowReferenceDetector(objectLongevityConfig, objectLongevityDetector);
        // Create a copy of the listeners, removing any duplicates
        this.refreshListeners = new CopyOnWriteArrayList<>(
                refreshListeners.stream().distinct().toArray(HollowConsumer.RefreshListener[]::new));
        this.apiFactory = apiFactory;
        this.hashCodeFinder = hashCodeFinder;
        this.doubleSnapshotConfig = doubleSnapshotConfig;
        this.objectLongevityConfig = objectLongevityConfig;
        this.staleReferenceDetector.startMonitoring();
        this.metrics = metrics;
        this.metricsCollector = metricsCollector;
        this.initialLoad = new CompletableFuture<>();
    }

    /**
     * Updates the client's state to the requested version, or to the version closest to but less than the requested version.
     *
     * @param requestedVersion the version to update the client to
     * @return true  if the update was either successfully completed and the updated version is the same as the requested version,
     *               or no updates were applied because the current version is the same as what the version would be after the updates were applied.
     *         false if the update completed but the client's updated version is not the same as the requested version, likely due to
     *               the requested version not being present in the data
     * @throws IllegalArgumentException if no data could be retrieved for that version or any earlier versions
     * @throws Throwable if any other exception occurred and the client could not be updated
     */
    /*
     * Note that this method is synchronized and it is the only method that modifies the
     * {@code hollowDataHolderVolatile}, so we don't need to worry about it changing out from
     * under us.
     */
    public synchronized boolean updateTo(long requestedVersion) throws Throwable {
        if (requestedVersion == getCurrentVersionId()) {
            if (requestedVersion == HollowConstants.VERSION_NONE && hollowDataHolderVolatile == null) {
                LOG.warning("No versions to update to, initializing to empty state");
                // attempting to refresh, but no available versions - initialize to empty state
                hollowDataHolderVolatile = newHollowDataHolder();
                forceDoubleSnapshotNextUpdate(); // intentionally ignore doubleSnapshotConfig
            }
            return true;
        }

        // Take a snapshot of the listeners to ensure additions or removals may occur concurrently
        // but will not take effect until a subsequent refresh
        final HollowConsumer.RefreshListener[] localListeners =
                refreshListeners.toArray(new HollowConsumer.RefreshListener[0]);

        long beforeVersion = getCurrentVersionId();

        for (HollowConsumer.RefreshListener listener : localListeners)
            listener.refreshStarted(beforeVersion, requestedVersion);

        try {
            HollowUpdatePlan updatePlan = shouldCreateSnapshotPlan()
                ? planner.planInitializingUpdate(requestedVersion)
                : planner.planUpdate(hollowDataHolderVolatile.getCurrentVersion(), requestedVersion,
                        doubleSnapshotConfig.allowDoubleSnapshot());

            for (HollowConsumer.RefreshListener listener : localListeners)
                if (listener instanceof HollowConsumer.TransitionAwareRefreshListener)
                    ((HollowConsumer.TransitionAwareRefreshListener)listener).transitionsPlanned(beforeVersion, requestedVersion, updatePlan.isSnapshotPlan(), updatePlan.getTransitionSequence());

            if (updatePlan.destinationVersion() == HollowConstants.VERSION_NONE
                    && requestedVersion != HollowConstants.VERSION_LATEST)
                throw new IllegalArgumentException(String.format("Could not create an update plan for version %s, because that version or any previous versions could not be retrieved.", requestedVersion));

            if (updatePlan.equals(HollowUpdatePlan.DO_NOTHING)
                    && requestedVersion == HollowConstants.VERSION_LATEST)
                throw new IllegalArgumentException("Could not create an update plan, because no existing versions could be retrieved.");

            if (updatePlan.destinationVersion(requestedVersion) == getCurrentVersionId())
                return true;

            if (updatePlan.isSnapshotPlan()) {
                if (hollowDataHolderVolatile == null || doubleSnapshotConfig.allowDoubleSnapshot()) {
                    hollowDataHolderVolatile = newHollowDataHolder();
                    hollowDataHolderVolatile.update(updatePlan, localListeners);
                    forceDoubleSnapshot = false;
                }
            } else {
                hollowDataHolderVolatile.update(updatePlan, localListeners);
            }

            for(HollowConsumer.RefreshListener refreshListener : localListeners)
                refreshListener.refreshSuccessful(beforeVersion, getCurrentVersionId(), requestedVersion);

            metrics.updateTypeStateMetrics(getStateEngine(), requestedVersion);
            if(metricsCollector != null)
                metricsCollector.collect(metrics);

            initialLoad.complete(getCurrentVersionId()); // only set the first time
            return getCurrentVersionId() == requestedVersion;
        } catch(Throwable th) {
            forceDoubleSnapshotNextUpdate();
            metrics.updateRefreshFailed();
            if(metricsCollector != null)
                metricsCollector.collect(metrics);
            for(HollowConsumer.RefreshListener refreshListener : localListeners)
                refreshListener.refreshFailed(beforeVersion, getCurrentVersionId(), requestedVersion, th);

            // intentionally omitting a call to initialLoad.completeExceptionally(th), for producers
            // that write often a consumer has a chance to try another snapshot that might succeed

            throw th;
        }
    }

    public synchronized void addRefreshListener(HollowConsumer.RefreshListener refreshListener,
            HollowConsumer c) {
        if (refreshListener instanceof HollowConsumer.RefreshRegistrationListener) {
            if (!refreshListeners.contains(refreshListener)) {
                ((HollowConsumer.RefreshRegistrationListener)refreshListener).onBeforeAddition(c);
            }
            refreshListeners.add(refreshListener);
        } else {
            refreshListeners.addIfAbsent(refreshListener);
        }
    }

    public synchronized void removeRefreshListener(HollowConsumer.RefreshListener refreshListener,
            HollowConsumer c) {
        if (refreshListeners.remove(refreshListener)) {
            if (refreshListener instanceof HollowConsumer.RefreshRegistrationListener) {
                ((HollowConsumer.RefreshRegistrationListener)refreshListener).onAfterRemoval(c);
            }
        }
    }

    public long getCurrentVersionId() {
        HollowDataHolder hollowDataHolderLocal = hollowDataHolderVolatile;
        return hollowDataHolderLocal != null ? hollowDataHolderLocal.getCurrentVersion()
            : HollowConstants.VERSION_NONE;
    }

    public void forceDoubleSnapshotNextUpdate() {
        this.forceDoubleSnapshot = true;
    }

    /**
     * Whether or not a snapshot plan should be created. Visible for testing.
     */
    boolean shouldCreateSnapshotPlan() {
        return getCurrentVersionId() == HollowConstants.VERSION_NONE
            ||  (forceDoubleSnapshot && doubleSnapshotConfig.allowDoubleSnapshot());
    }

    private HollowDataHolder newHollowDataHolder() {
        return new HollowDataHolder(newStateEngine(), apiFactory,
                failedTransitionTracker, staleReferenceDetector,
                objectLongevityConfig).setFilter(filter);
    }

    private HollowReadStateEngine newStateEngine() {
        HollowDataHolder hollowDataHolderLocal = hollowDataHolderVolatile;
        if (hollowDataHolderLocal != null) {
            ArraySegmentRecycler existingRecycler =
                    hollowDataHolderLocal.getStateEngine().getMemoryRecycler();
            return new HollowReadStateEngine(hashCodeFinder, true, existingRecycler);
        }
        return new HollowReadStateEngine(hashCodeFinder);
    }

    public StackTraceRecorder getStaleReferenceUsageStackTraceRecorder() {
        return staleReferenceDetector.getStaleReferenceStackTraceRecorder();
    }

    public HollowReadStateEngine getStateEngine() {
        HollowDataHolder hollowDataHolderLocal = hollowDataHolderVolatile;
        return hollowDataHolderLocal == null ? null : hollowDataHolderLocal.getStateEngine();
    }

    public HollowAPI getAPI() {
        HollowDataHolder hollowDataHolderLocal = hollowDataHolderVolatile;
        return hollowDataHolderLocal == null ? null : hollowDataHolderLocal.getAPI();
    }

    public void setFilter(HollowFilterConfig filter) {
        this.filter = filter;
    }

    /**
     * @return the number of failed snapshot transitions stored in the {@link FailedTransitionTracker}.
     */
    public int getNumFailedSnapshotTransitions() {
        return failedTransitionTracker.getNumFailedSnapshotTransitions();
    }

    /**
     * @return the number of failed delta transitions stored in the {@link FailedTransitionTracker}.
     */
    public int getNumFailedDeltaTransitions() {
        return failedTransitionTracker.getNumFailedDeltaTransitions();
    }

    /**
     * Clear any failed transitions from the {@link FailedTransitionTracker}, so that they may be reattempted when an update is triggered.
     */
    public void clearFailedTransitions() {
        this.failedTransitionTracker.clear();
    }

    /**
     * @return a future that will be completed with the version of data loaded when the initial load of data
     * has completed.
     */
    public CompletableFuture<Long> getInitialLoad() {
        return this.initialLoad;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.producer.metrics;

import com.netflix.hollow.api.producer.AbstractHollowProducerListener;
import com.netflix.hollow.api.producer.HollowProducer;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import java.time.Duration;
import java.util.OptionalLong;

/**
 * A class for computing Hollow Producer metrics, that requires extending subclasses to implement custom metrics reporting.
 * <p>
 * This class computes Hollow Producer metrics by listening to the stages of a producer's lifecycle. This class implements the
 * {@code ProducerMetricsReporting} interface which enforces concrete subclasses to implement custom metrics reporting behavior.
 */
public abstract class AbstractProducerMetricsListener extends AbstractHollowProducerListener implements
        ProducerMetricsReporting {

    private CycleMetrics.Builder cycleMetricsBuilder;
    private AnnouncementMetrics.Builder announcementMetricsBuilder;

    // visible for testing
    private long consecutiveFailures;
    OptionalLong lastCycleSuccessTimeNanoOptional;
    OptionalLong lastAnnouncementSuccessTimeNanoOptional;

    public AbstractProducerMetricsListener() {
        consecutiveFailures = 0l;
        lastCycleSuccessTimeNanoOptional = OptionalLong.empty();
        lastAnnouncementSuccessTimeNanoOptional = OptionalLong.empty();
    }

    @Override
    public void onCycleStart(long version) {
        cycleMetricsBuilder = new CycleMetrics.Builder();
    }

    @Override
    public void onAnnouncementStart(long version) {
        announcementMetricsBuilder = new AnnouncementMetrics.Builder();
    }

    /**
     * Reports metrics for when cycle is skipped due to reasons such as the producer not being the leader in a multiple-producer setting.
     * In a multiple producer setting, leader election typically favors long-lived leaders to avoid producer runs from frequently requiring
     * to reload the full state before publishing data. When a cycle is skipped because the producer wasn't primary, the current value of
     * no. of consecutive failures and most recent cycle success time are retained, and no cycle status (success or fail) is reported.
     * @param reason Reason why the run was skipped
     */
    @Override
    public void onCycleSkip(CycleSkipReason reason) {
        cycleMetricsBuilder.setConsecutiveFailures(consecutiveFailures);

        lastCycleSuccessTimeNanoOptional.ifPresent(cycleMetricsBuilder::setLastCycleSuccessTimeNano);

        // isCycleSuccess and cycleDurationMillis are not set for skipped cycles
        cycleMetricsBuilder.setConsecutiveFailures(consecutiveFailures);

        cycleMetricsReporting(cycleMetricsBuilder.build());
    }

    /**
     * Reports announcement-related metrics.
     * @param status Indicates whether the announcement succeeded of failed
     * @param readState Hollow data state that is being published, used in this method for computing data size
     * @param version Version of data that was announced
     * @param elapsed Announcement start to end duration
     */
    @Override
    public void onAnnouncementComplete(com.netflix.hollow.api.producer.Status status, HollowProducer.ReadState readState, long version, Duration elapsed) {
        boolean isAnnouncementSuccess = false;
        long dataSizeBytes = 0l;

        if (status.getType() == com.netflix.hollow.api.producer.Status.StatusType.SUCCESS) {
            isAnnouncementSuccess = true;
            lastAnnouncementSuccessTimeNanoOptional = OptionalLong.of(System.nanoTime());
        }

        HollowReadStateEngine stateEngine = readState.getStateEngine();
        dataSizeBytes = stateEngine.calcApproxDataSize();

        announcementMetricsBuilder
                .setDataSizeBytes(dataSizeBytes)
                .setIsAnnouncementSuccess(isAnnouncementSuccess)
                .setAnnouncementDurationMillis(elapsed.toMillis());

        lastAnnouncementSuccessTimeNanoOptional.ifPresent(announcementMetricsBuilder::setLastAnnouncementSuccessTimeNano);

        announcementMetricsReporting(announcementMetricsBuilder.build());
    }

    /**
     * On cycle completion this method reports cycle metrics.
     * @param status Whether the cycle succeeded or failed
     * @param readState Hollow data state published by cycle, not used here because data size is known from when announcement completed
     * @param version Version of data that was published in this cycle
     * @param elapsed Cycle start to end duration
     */
    @Override
    public void onCycleComplete(com.netflix.hollow.api.producer.Status status, HollowProducer.ReadState readState, long version, Duration elapsed) {
        boolean isCycleSuccess;
        long cycleEndTimeNano = System.nanoTime();

        if (status.getType() == com.netflix.hollow.api.producer.Status.StatusType.SUCCESS) {
            isCycleSuccess = true;
            consecutiveFailures = 0l;
            lastCycleSuccessTimeNanoOptional = OptionalLong.of(cycleEndTimeNano);
        } else {
            isCycleSuccess = false;
            consecutiveFailures ++;
        }

        cycleMetricsBuilder
                .setConsecutiveFailures(consecutiveFailures)
                .setCycleDurationMillis(elapsed.toMillis())
                .setIsCycleSuccess(isCycleSuccess);

        lastCycleSuccessTimeNanoOptional.ifPresent(cycleMetricsBuilder::setLastCycleSuccessTimeNano);

        cycleMetricsReporting(cycleMetricsBuilder.build());
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine;

import com.netflix.hollow.core.HollowBlobHeader;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.schema.HollowSchema;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Contains serialization logic for the hollow blob headers.
 *
 * @see HollowBlobHeader
 *
 */
public class HollowBlobHeaderReader {

    public HollowBlobHeader readHeader(InputStream is) throws IOException {
        HollowBlobHeader header = new HollowBlobHeader();
        DataInputStream dis = new DataInputStream(is);

        int headerVersion = dis.readInt();
        if(headerVersion != HollowBlobHeader.HOLLOW_BLOB_VERSION_HEADER) {
            throw new IOException("The HollowBlob you are trying to read is incompatible.  "
                    + "The expected Hollow blob version was " + HollowBlobHeader.HOLLOW_BLOB_VERSION_HEADER + " but the actual version was " + headerVersion);
        }

        header.setBlobFormatVersion(headerVersion);

        header.setOriginRandomizedTag(dis.readLong());
        header.setDestinationRandomizedTag(dis.readLong());

        int oldBytesToSkip = VarInt.readVInt(is); /// pre v2.2.0 envelope

        if(oldBytesToSkip != 0) {
            int numSchemas = VarInt.readVInt(is);

            List<HollowSchema> schemas = new ArrayList<HollowSchema>();
            for(int i=0;i<numSchemas;i++)
                schemas.add(HollowSchema.readFrom(is));
            header.setSchemas(schemas);

            int bytesToSkip = VarInt.readVInt(is); /// forwards-compatibility, new data can be added here.
            while(bytesToSkip > 0) {
                int skippedBytes = (int)is.skip(bytesToSkip);
                if(skippedBytes < 0)
                    throw new EOFException();
                bytesToSkip -= skippedBytes;
            }
        }

        Map<String, String> headerTags = readHeaderTags(dis);
        header.setHeaderTags(headerTags);

        return header;
    }

    /**
     * Map of string header tags reading.
     *
     * @param dis
     * @throws IOException
     */
    private Map<String, String> readHeaderTags(DataInputStream dis) throws IOException {
        int numHeaderTags = dis.readShort();
        Map<String, String> headerTags = new HashMap<String, String>();
        for (int i = 0; i < numHeaderTags; i++) {
            headerTags.put(dis.readUTF(), dis.readUTF());
        }
        return headerTags;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.write.objectmapper;

import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter;

import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;

public class HollowObjectMapper {

    private final HollowWriteStateEngine stateEngine;

    private final ConcurrentHashMap<Type, String> typeNameMappers;
    private final ConcurrentHashMap<String, HollowTypeMapper> typeMappers;

    private AtomicInteger unassignedTypeCounter = new AtomicInteger(0);

    private boolean ignoreListOrdering = false;
    private boolean useDefaultHashKeys = true;

    public HollowObjectMapper(HollowWriteStateEngine stateEngine) {
        this.stateEngine = stateEngine;
        this.typeNameMappers = new ConcurrentHashMap<>();
        this.typeMappers = new ConcurrentHashMap<>();
    }

    public void ignoreListOrdering() {
        this.ignoreListOrdering = true;
    }

    public void useDefaultHashKeys() {
        this.useDefaultHashKeys = true;
    }

    public void doNotUseDefaultHashKeys() {
        this.useDefaultHashKeys = false;
    }

    /**
     * Adds the specified POJO to the state engine.
     *

     * Unless previously initialized with {@link #initializeTypeState(Class)},
     * the first time an instance of a particular type is added
     * its schema is derived and added to the data model.
     *
     * @param o the POJO to add
     * @return the ordinal assigned to the newly added object
     */
    public int add(Object o) {
        HollowTypeMapper typeMapper = getTypeMapper(o.getClass(), null, null);
        return typeMapper.write(o);
    }

    /**
     * Warning: Experimental.  the FlatRecord feature is subject to breaking changes.
     */
    @Deprecated
    public void writeFlat(Object o, FlatRecordWriter flatRecordWriter) {
    	HollowTypeMapper typeMapper = getTypeMapper(o.getClass(), null, null);
    	typeMapper.writeFlat(o, flatRecordWriter);
    }

    /**
     * Extracts the primary key from the specified POJO.
     *
     * @param o the POJO
     * @return the primary key
     * @throws IllegalArgumentException if the POJO does not have primary key defined
     */
    public RecordPrimaryKey extractPrimaryKey(Object o) {
        HollowObjectTypeMapper typeMapper = (HollowObjectTypeMapper) getTypeMapper(o.getClass(), null, null);
        return new RecordPrimaryKey(typeMapper.getTypeName(), typeMapper.extractPrimaryKey(o));
    }

    /**
     * @param o the object to add
     * @return the ordinal assigned to the newly added object
     * @deprecated use {@link #add(Object)} instead.
     */
    @Deprecated
    public int addObject(Object o) {
        return add(o);
    }

    /**
     * Initializes the schema for the specified type in the data model.
     *

     * The schema will be derived from the field and type names in
     * clazz and added to the state engine's data model;
     * schemas of types referenced from clazz will also be added.
     * This can be used to add a type's schema to the state engine
     * without having to add any data for that type.
     *
     * @param clazz type whose schema to derive and add to the data model
     * @see #add(Object)
     */
    public void initializeTypeState(Class<?> clazz) {
        Objects.requireNonNull(clazz);
        getTypeMapper(clazz, null, null);
    }

    HollowTypeMapper getTypeMapper(Type type, String declaredName, String[] hashKeyFieldPaths) {
        return getTypeMapper(type, declaredName, hashKeyFieldPaths, -1, null);
    }

    HollowTypeMapper getTypeMapper(
            Type type, String declaredName, String[] hashKeyFieldPaths, int numShards, Set<Type> visited) {

        // Compute the type name
        String typeName = declaredName != null
                ? declaredName
                : typeNameMappers.computeIfAbsent(type, HollowObjectTypeMapper::getDefaultTypeName);
        HollowTypeMapper typeMapper = typeMappers.get(typeName);

        if (typeMapper == null) {
            if (visited == null) {
                // Used to detect circular references in the model
                // See HollowObjectTypeMapper and MappedField
                visited = new HashSet<>();
            }

            if (type instanceof ParameterizedType) {
                ParameterizedType parameterizedType = (ParameterizedType) type;
                Class<?> clazz = (Class<?>) parameterizedType.getRawType();

                if (List.class.isAssignableFrom(clazz)) {
                    typeMapper = new HollowListTypeMapper(this, parameterizedType, typeName, numShards,
                            ignoreListOrdering, visited);
                } else if (Set.class.isAssignableFrom(clazz)) {
                    typeMapper = new HollowSetTypeMapper(this, parameterizedType, typeName, hashKeyFieldPaths,
                            numShards, stateEngine, useDefaultHashKeys, visited);
                } else if (Map.class.isAssignableFrom(clazz)) {
                    typeMapper = new HollowMapTypeMapper(this, parameterizedType, typeName, hashKeyFieldPaths,
                            numShards, stateEngine, useDefaultHashKeys, visited);
                } else {
                    typeMapper = new HollowObjectTypeMapper(this, clazz, typeName, visited);
                }
            } else {
                typeMapper = new HollowObjectTypeMapper(this, (Class<?>) type, typeName, visited);
            }

            HollowTypeMapper existing = typeMappers.putIfAbsent(typeName, typeMapper);
            if (existing != null) {
                typeMapper = existing;
            } else {
                typeMapper.addTypeState(stateEngine);
            }
        }

        return typeMapper;
    }

    int nextUnassignedTypeId() {
        return unassignedTypeCounter.getAndIncrement();
    }

    public HollowWriteStateEngine getStateEngine() {
        return stateEngine;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.memory;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.BitSet;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;

/**
 * This is a lock-free, thread-safe version of a {@link java.util.BitSet}.

 *
 * Instead of a long array to hold the bits, this implementation uses an AtomicLongArray, then
 * does the appropriate compare-and-swap operations when setting the bits.
 *
 * @author dkoszewnik
 *
 */
public class ThreadSafeBitSet {

    public static final int DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS = 14;

    private final int numLongsPerSegment;
    private final int log2SegmentSize;
    private final int segmentMask;
    private final AtomicReference<ThreadSafeBitSetSegments> segments;

    public ThreadSafeBitSet() {
        this(DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS); /// 16384 bits, 2048 bytes, 256 longs per segment
    }

    public ThreadSafeBitSet(int log2SegmentSizeInBits) {
        this(log2SegmentSizeInBits, 0);
    }

    public ThreadSafeBitSet(int log2SegmentSizeInBits, int numBitsToPreallocate) {
        if(log2SegmentSizeInBits < 6)
            throw new IllegalArgumentException("Cannot specify fewer than 64 bits in each segment!");

        this.log2SegmentSize = log2SegmentSizeInBits;
        this.numLongsPerSegment = (1 << (log2SegmentSizeInBits - 6));
        this.segmentMask = numLongsPerSegment - 1;

        long numBitsPerSegment = numLongsPerSegment * 64;
        int numSegmentsToPreallocate = numBitsToPreallocate == 0 ? 1 : (int)(((numBitsToPreallocate - 1) / numBitsPerSegment) + 1);

        segments = new AtomicReference<ThreadSafeBitSetSegments>();
        segments.set(new ThreadSafeBitSetSegments(numSegmentsToPreallocate, numLongsPerSegment));
    }

    public void set(int position) {
        int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
        int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
        int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)

        AtomicLongArray segment = getSegment(segmentPosition);

        long mask = 1L << bitPosition;

        // Thread safety: we need to loop until we win the race to set the long value.
        while(true) {
            // determine what the new long value will be after we set the appropriate bit.
            long currentLongValue = segment.get(longPosition);
            long newLongValue = currentLongValue | mask;

            // if no other thread has modified the value since we read it, we won the race and we are done.
            if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
                break;
        }
    }

    public void clear(int position) {
        int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
        int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
        int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)

        AtomicLongArray segment = getSegment(segmentPosition);

        long mask = ~(1L << bitPosition);

        // Thread safety: we need to loop until we win the race to set the long value.
        while(true) {
            // determine what the new long value will be after we set the appropriate bit.
            long currentLongValue = segment.get(longPosition);
            long newLongValue = currentLongValue & mask;

            // if no other thread has modified the value since we read it, we won the race and we are done.
            if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
                break;
        }
    }

    public boolean get(int position) {
        int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
        int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
        int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)

        AtomicLongArray segment = getSegment(segmentPosition);

        long mask = 1L << bitPosition;

        return ((segment.get(longPosition) & mask) != 0);
    }

    public long maxSetBit() {
        ThreadSafeBitSetSegments segments = this.segments.get();

        int segmentIdx = segments.numSegments() - 1;

        for(;segmentIdx >= 0; segmentIdx--) {
            AtomicLongArray segment = segments.getSegment(segmentIdx);
            for(int longIdx=segment.length() - 1; longIdx >= 0; longIdx--) {
                long l = segment.get(longIdx);
                if(l != 0)
                    return (segmentIdx << log2SegmentSize) + (longIdx * 64) + (63 - Long.numberOfLeadingZeros(l));
            }
        }

        return -1;
    }

    public int nextSetBit(int fromIndex) {
        if (fromIndex < 0)
            throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex);

        int segmentPosition = fromIndex >>> log2SegmentSize; /// which segment -- div by num bits per segment

        ThreadSafeBitSetSegments segments = this.segments.get();

        if(segmentPosition >= segments.numSegments())
            return -1;

        int longPosition = (fromIndex >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
        int bitPosition = fromIndex & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
        AtomicLongArray segment = segments.getSegment(segmentPosition);

        long word = segment.get(longPosition) & (0xffffffffffffffffL << bitPosition);

        while (true) {
            if (word != 0)
                return (segmentPosition << (log2SegmentSize)) + (longPosition << 6) + Long.numberOfTrailingZeros(word);
            if (++longPosition > segmentMask) {
                segmentPosition++;
                if(segmentPosition >= segments.numSegments())
                    return -1;
                segment = segments.getSegment(segmentPosition);
                longPosition = 0;
            }

            word = segment.get(longPosition);
        }
    }

    /**
     * @return the number of bits which are set in this bit set.
     */
    public int cardinality() {
        ThreadSafeBitSetSegments segments = this.segments.get();

        int numSetBits = 0;

        for(int i=0;i<segments.numSegments();i++) {
            AtomicLongArray segment = segments.getSegment(i);
            for(int j=0;j<segment.length();j++) {
                numSetBits += Long.bitCount(segment.get(j));
            }
        }

        return numSetBits;
    }

    /**
     * @return the number of bits which are current specified by this bit set.  This is the maximum value
     * to which you might need to iterate, if you were to iterate over all bits in this set.
     */
    public int currentCapacity() {
        return segments.get().numSegments() * (1 << log2SegmentSize);
    }

    /**
     * Clear all bits to 0.
     */
    public void clearAll() {
        ThreadSafeBitSetSegments segments = this.segments.get();

        for(int i=0;i<segments.numSegments();i++) {
            AtomicLongArray segment = segments.getSegment(i);

            for(int j=0;j<segment.length();j++) {
                segment.set(j, 0L);
            }
        }
    }

    /**
     * Return a new bit set which contains all bits which are contained in this bit set, and which are NOT contained in the other bit set.

     *
     * In other words, return a new bit set, which is a bitwise and with the bitwise not of the other bit set.
     *
     * @param other the other bit set
     * @return the resulting bit set
     */
    public ThreadSafeBitSet andNot(ThreadSafeBitSet other) {
        if(other.log2SegmentSize != log2SegmentSize)
            throw new IllegalArgumentException("Segment sizes must be the same");

        ThreadSafeBitSetSegments thisSegments = this.segments.get();
        ThreadSafeBitSetSegments otherSegments = other.segments.get();
        ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(thisSegments.numSegments(), numLongsPerSegment);

        for(int i=0;i<thisSegments.numSegments();i++) {
            AtomicLongArray thisArray = thisSegments.getSegment(i);
            AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
            AtomicLongArray newArray = newSegments.getSegment(i);

            for(int j=0;j<thisArray.length();j++) {
                long thisLong = thisArray.get(j);
                long otherLong = (otherArray == null) ? 0 : otherArray.get(j);

                newArray.set(j, thisLong & ~otherLong);
            }
        }

        ThreadSafeBitSet andNot = new ThreadSafeBitSet(log2SegmentSize);
        andNot.segments.set(newSegments);
        return andNot;
    }

    /**
     * Return a new bit set which contains all bits which are contained in *any* of the specified bit sets.
     *
     * @param bitSets the other bit sets
     * @return the resulting bit set
     */
    public static ThreadSafeBitSet orAll(ThreadSafeBitSet... bitSets) {
        if(bitSets.length == 0)
            return new ThreadSafeBitSet();

        int log2SegmentSize = bitSets[0].log2SegmentSize;
        int numLongsPerSegment = bitSets[0].numLongsPerSegment;

        ThreadSafeBitSetSegments segments[] = new ThreadSafeBitSetSegments[bitSets.length];
        int maxNumSegments = 0;

        for(int i=0;i<bitSets.length;i++) {
            if(bitSets[i].log2SegmentSize != log2SegmentSize)
                throw new IllegalArgumentException("Segment sizes must be the same");

            segments[i] = bitSets[i].segments.get();
            if(segments[i].numSegments() > maxNumSegments)
                maxNumSegments = segments[i].numSegments();
        }

        ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(maxNumSegments, numLongsPerSegment);

        AtomicLongArray segment[] = new AtomicLongArray[segments.length];

        for(int i=0;i<maxNumSegments;i++) {
            for(int j=0;j<segments.length;j++) {
                segment[j] = i < segments[j].numSegments() ? segments[j].getSegment(i) : null;
            }

            AtomicLongArray newSegment = newSegments.getSegment(i);

            for(int j=0;j<numLongsPerSegment;j++) {
                long value = 0;
                for(int k=0;k<segments.length;k++) {
                    if(segment[k] != null)
                        value |= segment[k].get(j);
                }
                newSegment.set(j, value);
            }
        }

        ThreadSafeBitSet or = new ThreadSafeBitSet(log2SegmentSize);
        or.segments.set(newSegments);
        return or;
    }

    /**
     * Get the segment at segmentIndex.  If this segment does not yet exist, create it.
     *
     * @param segmentIndex the segment index
     * @return the segment
     */
    private AtomicLongArray getSegment(int segmentIndex) {
        ThreadSafeBitSetSegments visibleSegments = segments.get();

        while(visibleSegments.numSegments() <= segmentIndex) {
            /// Thread safety:  newVisibleSegments contains all of the segments from the currently visible segments, plus extra.
            /// all of the segments in the currently visible segments are canonical and will not change.
            ThreadSafeBitSetSegments newVisibleSegments = new ThreadSafeBitSetSegments(visibleSegments, segmentIndex + 1, numLongsPerSegment);

            /// because we are using a compareAndSet, if this thread "wins the race" and successfully sets this variable, then the segments
            /// which are newly defined in newVisibleSegments become canonical.
            if(segments.compareAndSet(visibleSegments, newVisibleSegments)) {
                visibleSegments = newVisibleSegments;
            } else {
                /// If we "lose the race" and are growing the ThreadSafeBitSet segments larger,
                /// then we will gather the new canonical sets from the update which we missed on the next iteration of this loop.
                /// Newly defined segments in newVisibleSegments will be discarded, they do not get to become canonical.
                visibleSegments = segments.get();
            }
        }

        return visibleSegments.getSegment(segmentIndex);
    }

    private static class ThreadSafeBitSetSegments {

        private final AtomicLongArray segments[];

        private ThreadSafeBitSetSegments(int numSegments, int segmentLength) {
            AtomicLongArray segments[] = new AtomicLongArray[numSegments];

            for(int i=0;i<numSegments;i++) {
                segments[i] = new AtomicLongArray(segmentLength);
            }

            /// Thread safety: Because this.segments is final, the preceding operations in this constructor are guaranteed to be visible to any
            /// other thread which accesses this.segments.
            this.segments = segments;
        }

        private ThreadSafeBitSetSegments(ThreadSafeBitSetSegments copyFrom, int numSegments, int segmentLength) {
            AtomicLongArray segments[] = new AtomicLongArray[numSegments];

            for(int i=0;i<numSegments;i++) {
                segments[i] = i < copyFrom.numSegments() ? copyFrom.getSegment(i) : new AtomicLongArray(segmentLength);
            }

            /// see above re: thread-safety of this assignment
            this.segments = segments;
        }

        public int numSegments() {
            return segments.length;
        }

        public AtomicLongArray getSegment(int index) {
            return segments[index];
        }

    }

    public void serializeBitsTo(DataOutputStream os) throws IOException {
        ThreadSafeBitSetSegments segments = this.segments.get();

        os.writeInt(segments.numSegments() * numLongsPerSegment);

        for(int i=0;i<segments.numSegments();i++) {
            AtomicLongArray arr = segments.getSegment(i);

            for(int j=0;j<arr.length();j++) {
                os.writeLong(arr.get(j));
            }
        }
    }

    @Override
    public boolean equals(Object obj) {
        if(!(obj instanceof ThreadSafeBitSet))
            return false;

        ThreadSafeBitSet other = (ThreadSafeBitSet)obj;

        if(other.log2SegmentSize != log2SegmentSize)
            throw new IllegalArgumentException("Segment sizes must be the same");

        ThreadSafeBitSetSegments thisSegments = this.segments.get();
        ThreadSafeBitSetSegments otherSegments = other.segments.get();

        for(int i=0;i<thisSegments.numSegments();i++) {
            AtomicLongArray thisArray = thisSegments.getSegment(i);
            AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;

            for(int j=0;j<thisArray.length();j++) {
                long thisLong = thisArray.get(j);
                long otherLong = (otherArray == null) ? 0 : otherArray.get(j);

                if(thisLong != otherLong)
                    return false;
            }
        }

        for(int i=thisSegments.numSegments();i<otherSegments.numSegments();i++) {
            AtomicLongArray otherArray = otherSegments.getSegment(i);

            for(int j=0;j<otherArray.length();j++) {
                long l = otherArray.get(j);

                if(l != 0)
                    return false;
            }
        }

        return true;
    }

    /**
     * @return a new BitSet with same bits set
     */
    public BitSet toBitSet() {
        BitSet resultSet = new BitSet();
        int ordinal = this.nextSetBit(0);
        while(ordinal!=-1) {
            resultSet.set(ordinal);
            ordinal = this.nextSetBit(ordinal + 1);
        }
        return resultSet;
    }

    @Override
    public String toString() {
        return toBitSet().toString();
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.metrics;

import com.netflix.hollow.api.producer.HollowProducer;
import com.netflix.hollow.api.producer.HollowProducerListener;
import com.netflix.hollow.api.producer.Status;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import java.util.concurrent.atomic.AtomicInteger;

public class HollowProducerMetrics extends HollowMetrics {
    private int cyclesCompleted = 0;
    private int cyclesSucceeded = 0;
    private int cycleFailed = 0;
    // Snapshots can be published asynchronously resulting concurrent
    // access to the snapshot metrics
    private AtomicInteger snapshotsCompleted = new AtomicInteger();
    private AtomicInteger snapshotsFailed = new AtomicInteger();
    private int deltasCompleted = 0;
    private int deltasFailed = 0;
    private int reverseDeltasCompleted = 0;
    private int reverseDeltasFailed = 0;

    /**
     * Updates the producer metrics:
     * cycles completed, version and type's footprint and ordinals.
     * @param producerStatus the producer status
     */
    public void updateCycleMetrics(HollowProducerListener.ProducerStatus producerStatus) {
        Status.StatusType st = producerStatus.getStatus() == HollowProducerListener.Status.SUCCESS
                ? Status.StatusType.SUCCESS
                : Status.StatusType.FAIL;

        updateCycleMetrics(new Status(st, producerStatus.getCause()), producerStatus.getReadState(), producerStatus.getVersion());
    }

    /**
     * Updates the producer metrics:
     * cycles completed, version and type's footprint and ordinals.
     * @param status the status
     * @param readState the read state
     * @param version the version
     */
    public void updateCycleMetrics(Status status, HollowProducer.ReadState readState, long version) {
        cyclesCompleted++;
        if(status.getType() == Status.StatusType.FAIL) {
            cycleFailed++;
            return;
        }
        cyclesSucceeded++;

        if(readState != null) {
            HollowReadStateEngine hollowReadStateEngine = readState.getStateEngine();
            super.update(hollowReadStateEngine, version);
        } else {
            super.update(version);
        }
    }

    public void updateBlobTypeMetrics(HollowProducerListener.PublishStatus publishStatus) {
        Status.StatusType st = publishStatus.getStatus() == HollowProducerListener.Status.SUCCESS
                ? Status.StatusType.SUCCESS
                : Status.StatusType.FAIL;

        updateBlobTypeMetrics(new Status(st, publishStatus.getCause()), publishStatus.getBlob());
    }

    public void updateBlobTypeMetrics(Status status, HollowProducer.Blob blob) {
        HollowProducer.Blob.Type blobType = blob.getType();
        switch (blobType) {
            case SNAPSHOT:
                if(status.getType() == Status.StatusType.SUCCESS)
                    snapshotsCompleted.incrementAndGet();
                else
                    snapshotsFailed.incrementAndGet();
                break;
            case DELTA:
                if(status.getType() == Status.StatusType.SUCCESS)
                    deltasCompleted++;
                else
                    deltasFailed++;
                break;
            case REVERSE_DELTA:
                if(status.getType() == Status.StatusType.SUCCESS)
                    reverseDeltasCompleted++;
                else
                    reverseDeltasFailed++;
                break;
        }
    }

    public int getCyclesCompleted() {
        return this.cyclesCompleted;
    }

    public int getCyclesSucceeded() {
        return this.cyclesSucceeded;
    }

    public int getCycleFailed() {
        return this.cycleFailed;
    }

    public int getSnapshotsCompleted() {
        return snapshotsCompleted.get();
    }

    public int getSnapshotsFailed() {
        return snapshotsFailed.get();
    }

    public int getDeltasCompleted() {
        return deltasCompleted;
    }

    public int getDeltasFailed() {
        return deltasFailed;
    }

    public int getReverseDeltasCompleted() {
        return reverseDeltasCompleted;
    }

    public int getReverseDeltasFailed() {
        return reverseDeltasFailed;
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.codegen;

import static com.netflix.hollow.api.codegen.HollowCodeGenerationUtils.isPrimitiveType;

import com.netflix.hollow.core.HollowDataset;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.util.HollowWriteStateCreator;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;

/**
 * This class is used to generate java code which defines POJOs, which can in turn be used to populate a
 * {@link HollowWriteStateEngine} via a {@link HollowObjectMapper}
 *
 * The generated java code is based on a data model (defined by a set of {@link HollowSchema}).
 *
 * You may also run the main() method directly.
 */
public class HollowPOJOGenerator {
    /**
     * An enumeration of possible arguments to the code generator when being called via the main
     * function. Not expected to be used outside the library itself, except for documentation
     * purposes.
     * Unless otherwise noted, having repeated parameters results in the previous value being
     * overwritten.
     */
    public enum GeneratorArguments {
        /**
         * Add a class to the data model. Takes the fully qualified class name. This class must be
         * available on the classpath. Having multiple of this parameter results in multiple classes
         * being added to the data model.
         */
        addToDataModel,
        /**
         * Add schema from a schema file to the data model. The schema file must be available on the
         * classpath. Having multiple of this parameter results in multiple schemas being added to
         * the data model.
         */
        addSchemaFileToDataModel,
        /**
         * Sets the path the files with be generated in.
         */
        pathToGeneratedFiles,
        /**
         * Sets the package name for the generated files.
         */
        packageName,
        /**
         * Sets the suffix for the generated POJO class names.
         */
        pojoClassNameSuffix;
    }

    private final String packageName;
    private final String pojoClassNameSuffix;
    private final HollowDataset dataset;

    public HollowPOJOGenerator(String packageName, String pojoClassNameSuffix, HollowDataset dataset) {
        this.packageName = packageName;
        this.pojoClassNameSuffix = pojoClassNameSuffix;
        this.dataset = dataset;
    }

    /**
     * Usage: java HollowPOJOGenerator --argName1=argValue1 --argName2==argValue2. See {@link GeneratorArguments}
     * for available arguments.
     * @param args the arguments
     * @throws IOException if the POJOs cannot be created
     * @throws ClassNotFoundException if the class for a data type cannot be loaded
     */
    public static void main(String[] args) throws IOException, ClassNotFoundException {
        if (args.length == 0) {
            System.out.println("Usage:\n"
                    + "java " + HollowPOJOGenerator.class.getName() + " --arg1=value1 --arg2=value2\n"
                    + "see " + GeneratorArguments.class.getName() + " for available arguments.");
            return;
        }
        HollowWriteStateEngine engine = new HollowWriteStateEngine();
        String packageName = null;
        String pojoClassNameSuffix = null;
        String pathToGeneratedFiles = null;
        HollowObjectMapper mapper = new HollowObjectMapper(engine);
        ArgumentParser<GeneratorArguments> argumentParser = new ArgumentParser(GeneratorArguments.class, args);
        for (ArgumentParser<GeneratorArguments>.ParsedArgument arg : argumentParser.getParsedArguments()) {
            switch (arg.getKey()) {
                case addToDataModel:
                    mapper.initializeTypeState(HollowPOJOGenerator.class.getClassLoader().loadClass(arg.getValue()));
                    break;
                case addSchemaFileToDataModel:
                    HollowWriteStateCreator.readSchemaFileIntoWriteState(arg.getValue(), engine);
                    break;
                case pathToGeneratedFiles:
                    pathToGeneratedFiles = arg.getValue();
                    break;
                case packageName:
                    packageName = arg.getValue();
                    break;
                case pojoClassNameSuffix:
                    pojoClassNameSuffix = arg.getValue();
                    break;
                default:
                    throw new IllegalArgumentException("Unhandled argument " + arg.getKey());
            }
        }
        new HollowPOJOGenerator(packageName, pojoClassNameSuffix, engine).generateFiles(pathToGeneratedFiles);
    }

    public void generateFiles(String directory) throws IOException {
        generateFiles(new File(directory));
    }

    public void generateFiles(File directory) throws IOException {
        Path destinationPath = directory.toPath();
        Path packagePath = Paths.get(packageName.replace(".", File.separator));
        if (!destinationPath.toAbsolutePath().endsWith(packagePath)) {
            destinationPath = destinationPath.resolve(packagePath);
        }
        directory = destinationPath.toFile();
        if (!directory.exists()) directory.mkdirs();
        for (HollowSchema schema : dataset.getSchemas()) {
            if (schema instanceof HollowObjectSchema && !isPrimitiveType(schema.getName())) {
                HollowPOJOClassGenerator generator = new HollowPOJOClassGenerator(dataset, (HollowObjectSchema) schema,
                        packageName, pojoClassNameSuffix);
                FileWriter writer = new FileWriter(new File(directory, generator.getClassName() + ".java"));
                writer.write(generator.generate());
                writer.close();
            }
        }
    }
}