/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.codegen;

import com.netflix.hollow.api.codegen.api.HollowDataAccessorGenerator;
import com.netflix.hollow.api.codegen.api.TypeAPIListJavaGenerator;
import com.netflix.hollow.api.codegen.api.TypeAPIMapJavaGenerator;
import com.netflix.hollow.api.codegen.api.TypeAPIObjectJavaGenerator;
import com.netflix.hollow.api.codegen.api.TypeAPISetJavaGenerator;
import com.netflix.hollow.api.codegen.delegate.HollowObjectDelegateCachedImplGenerator;
import com.netflix.hollow.api.codegen.delegate.HollowObjectDelegateInterfaceGenerator;
import com.netflix.hollow.api.codegen.delegate.HollowObjectDelegateLookupImplGenerator;
import com.netflix.hollow.api.codegen.indexes.HollowHashIndexGenerator;
import com.netflix.hollow.api.codegen.indexes.HollowPrimaryKeyIndexGenerator;
import com.netflix.hollow.api.codegen.indexes.HollowUniqueKeyIndexGenerator;
import com.netflix.hollow.api.codegen.indexes.LegacyHollowPrimaryKeyIndexGenerator;
import com.netflix.hollow.api.codegen.objects.HollowFactoryJavaGenerator;
import com.netflix.hollow.api.codegen.objects.HollowListJavaGenerator;
import com.netflix.hollow.api.codegen.objects.HollowMapJavaGenerator;
import com.netflix.hollow.api.codegen.objects.HollowObjectJavaGenerator;
import com.netflix.hollow.api.codegen.objects.HollowSetJavaGenerator;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.core.HollowDataset;
import com.netflix.hollow.core.schema.HollowListSchema;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchema.SchemaType;
import com.netflix.hollow.core.schema.HollowSetSchema;
import com.netflix.hollow.core.util.HollowWriteStateCreator;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.Set;

/**
 * This class is used to generate java code which defines an implementation of a {@link HollowAPI}.
 *
 * The generated java code is based on a data model (defined by a set of {@link HollowSchema}), and will
 * contain convenience methods for traversing a dataset, based on the specific fields in the data model.
 *
 * You may also run the main() method directly.
 */
public class HollowAPIGenerator {
    /**
     * An enumeration of possible arguments to the code generator when being called via the main
     * function. Not expected to be used outside the library itself, except for documentation
     * purposes.
     * Unless otherwise noted, having repeated parameters results in the previous value being
     * overwritten.
     */
    public enum GeneratorArguments {
        /**
         * Add a class to the data model. Takes the fully qualified class name. This class must be
         * available on the classpath. Having multiple of this parameter results in multiple classes
         * being added to the data model.
         */
        addToDataModel,
        /**
         * Add schema from a schema file to the data model. The schema file must be available on the
         * classpath. Having multiple of this parameter results in multiple schemas being added to
         * the data model.
         */
        addSchemaFileToDataModel,
        /**
         * Sets the API class name.
         */
        apiClassName,
        /**
         * Sets the class postfix for the generated types.
         */
        classPostfix,
        /**
         * Sets the getter prefix to the provided string.
         */
        getterPrefix,
        /**
         * Sets the package name for the generated files.
         */
        packageName,
        /**
         * Sets the path the files with be generated in.
         */
        pathToGeneratedFiles,
        /**
         * Parameterizes all methods that return a HollowObject.
         */
        parameterizeAllClassNames;
    }

    protected final String apiClassname;
    protected final String packageName;
    protected final Path destinationPath;
    protected final HollowDataset dataset;
    protected final Set<String> parameterizedTypes;
    protected final boolean parameterizeClassNames;
    protected final boolean hasCollectionsInDataSet;
    protected final HollowErgonomicAPIShortcuts ergonomicShortcuts;

    protected CodeGeneratorConfig config = new CodeGeneratorConfig("Hollow", "_"); // NOTE: to be backwards compatible

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     *
     * @deprecated use {@link #HollowAPIGenerator(String, String, HollowDataset, Path)} and use {@link #generateSourceFiles()}
     */
    public HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset) {
        this(apiClassname, packageName, dataset, Collections.<String>emptySet(), false, false);
    }

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     * @param destinationPath the directory under which the source files will be generated
     */
    public HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset, Path destinationPath) {
        this(apiClassname, packageName, dataset, Collections.<String>emptySet(), false, false, destinationPath);
    }

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     * @param parameterizeAllClassNames if true, all methods which return a Hollow Object will be parameterized.  This is useful when
     *                               alternate implementations are desired for some types.
     *
     * @deprecated use {@link #HollowAPIGenerator(String, String, HollowDataset, boolean, Path)} and use {@link #generateSourceFiles()}
     */
    public HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset, boolean parameterizeAllClassNames) {
        this(apiClassname, packageName, dataset, Collections.<String>emptySet(), parameterizeAllClassNames, false);
    }

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     * @param parameterizeAllClassNames if true, all methods which return a Hollow Object will be parameterized.  This is useful when
     *                               alternate implementations are desired for some types.
     * @param destinationPath the directory under which the source files will be generated
     */
    public HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset, boolean parameterizeAllClassNames, Path destinationPath) {
        this(apiClassname, packageName, dataset, Collections.<String>emptySet(), parameterizeAllClassNames, false, destinationPath);
    }

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     * @param parameterizeSpecificTypeNames methods with matching names which return a Hollow Object will be parameterized.  This is useful when
     *                               alternate implementations are desired for some types.
     *
     * @deprecated use {@link #HollowAPIGenerator(String, String, HollowDataset, Set, Path)} and use {@link #generateSourceFiles()}
     */
    public HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset, Set<String> parameterizeSpecificTypeNames) {
        this(apiClassname, packageName, dataset, parameterizeSpecificTypeNames, false, false);
    }

    /**
     * @param apiClassname the class name of the generated implementation of {@link HollowAPI}
     * @param packageName the package name under which all generated classes will be placed
     * @param dataset a HollowStateEngine containing the schemas which define the data model.
     * @param parameterizedTypes the parameterized types
     * @param destinationPath the directory under which the source files will be generated
     */
    public HollowAPIGenerator(String apiClassname,
                              String packageName,
                              HollowDataset dataset,
                              Set<String> parameterizedTypes,
                              Path destinationPath) {
        this(apiClassname, packageName, dataset, parameterizedTypes, false, false, destinationPath);
    }

    /**
     * @param apiClassname the api class name
     * @param packageName the api package name
     * @param dataset the data set
     * @param parameterizedTypes the set of parameterized types
     * @param parameterizeAllClassNames true if class names should be parameterized
     * @param useErgonomicShortcuts true if ergonomic shortcuts should be used
     * @deprecated construct with a {@code destinationPath} and use {@link #generateSourceFiles()}
     */
    protected HollowAPIGenerator(String apiClassname, String packageName, HollowDataset dataset, Set<String> parameterizedTypes, boolean parameterizeAllClassNames, boolean useErgonomicShortcuts) {
        this(apiClassname, packageName, dataset, parameterizedTypes, false, false, null);
    }

    protected HollowAPIGenerator(String apiClassname,
                                 String packageName,
                                 HollowDataset dataset,
                                 Set<String> parameterizedTypes,
                                 boolean parameterizeAllClassNames,
                                 boolean useErgonomicShortcuts,
                                 Path destinationPath) {
        this.apiClassname = apiClassname;
        this.packageName = packageName;
        this.dataset = dataset;
        this.hasCollectionsInDataSet = hasCollectionsInDataSet(dataset);
        this.parameterizedTypes = parameterizedTypes;
        this.parameterizeClassNames = parameterizeAllClassNames;
        this.ergonomicShortcuts = useErgonomicShortcuts ? new HollowErgonomicAPIShortcuts(dataset) : HollowErgonomicAPIShortcuts.NO_SHORTCUTS;

        if (destinationPath != null && packageName != null && !packageName.trim().isEmpty()) {
            Path packagePath = Paths.get(packageName.replace(".", File.separator));
            if (!destinationPath.toAbsolutePath().endsWith(packagePath)) {
                destinationPath = destinationPath.resolve(packagePath);
            }
        }
        this.destinationPath = destinationPath;
    }

    /**
     * Usage: java HollowAPIGenerator --argName1=argValue1 --argName2==argValue2. See {@link GeneratorArguments} for
     * available arguments.
     * @param args the arguments
     * @throws IOException if a schema could not be read or the data model written
     * @throws ClassNotFoundException if a class of a data model cannot be loaded
     */
    public static void main(String[] args) throws IOException, ClassNotFoundException {
        if (args.length == 0) {
            System.out.println("Usage:\n"
                    + "java " + HollowAPIGenerator.class.getName() + " --arg1=value1 --arg2=value2\n"
                    + "see " + GeneratorArguments.class.getName() + " for available arguments.");
            return;
        }
        HollowWriteStateEngine engine = new HollowWriteStateEngine();
        HollowAPIGenerator.Builder builder = new HollowAPIGenerator.Builder();
        HollowObjectMapper mapper = new HollowObjectMapper(engine);
        ArgumentParser<GeneratorArguments> argumentParser = new ArgumentParser(GeneratorArguments.class, args);
        for (ArgumentParser<GeneratorArguments>.ParsedArgument arg : argumentParser.getParsedArguments()) {
            switch (arg.getKey()) {
                case addToDataModel:
                    mapper.initializeTypeState(HollowAPIGenerator.class.getClassLoader().loadClass(arg.getValue()));
                    break;
                case addSchemaFileToDataModel:
                    HollowWriteStateCreator.readSchemaFileIntoWriteState(arg.getValue(), engine);
                    break;
                case apiClassName:
                    builder.withAPIClassname(arg.getValue());
                    break;
                case classPostfix:
                    builder.withClassPostfix(arg.getValue());
                    break;
                case getterPrefix:
                    builder.withGetterPrefix(arg.getValue());
                    break;
                case packageName:
                    builder.withPackageName(arg.getValue());
                    break;
                case pathToGeneratedFiles:
                    builder.withDestination(arg.getValue());
                    break;
                case parameterizeAllClassNames:
                    builder.withParameterizeAllClassNames(Boolean.valueOf(arg.getValue()));
                    break;
                default:
                    throw new IllegalArgumentException("Unhandled argument " + arg.getKey());
            }
        }
        builder.withDataModel(engine).build().generateSourceFiles();
    }

    /**
     * Determines whether DataSet contains any collections schema
     * @param dataset the data set
     * @return {@code true} if the data set contains any collections schema
     */
    protected static boolean hasCollectionsInDataSet(HollowDataset dataset) {
        for(HollowSchema schema : dataset.getSchemas()) {
            if ((schema instanceof HollowListSchema) ||
                    (schema instanceof HollowSetSchema) ||
                    (schema instanceof HollowMapSchema)) {
                return true;
            }
        }
        return false;
    }

    /**
     * Set the CodeGeneratorConfig
     * @param config the configuration
     */
    protected void setCodeGeneratorConfig(CodeGeneratorConfig config) {
        this.config = config;
    }

    /**
     * Use this method to override the default postfix "Hollow" for all generated Hollow object classes.
     * @param classPostfix the postfix for all generated Hollow object classes
     */
    public void setClassPostfix(String classPostfix) {
        config.setClassPostfix(classPostfix);
    }

    /**
     * Use this method to override the default prefix "_" for all getters on all generated Hollow object classes.
     * @param getterPrefix the prefix for all generated getters
     */
    public void setGetterPrefix(String getterPrefix) {
        config.setGetterPrefix(getterPrefix);
    }

    /**
     * Use this method to override generated classnames for type names corresponding to any class in the java.lang package.
     *
     * Defaults to false, which overrides only type names corresponding to a few select classes in java.lang.
     *
     * @param useAggressiveSubstitutions true if applied.
     */
    public void setUseAggressiveSubstitutions(boolean useAggressiveSubstitutions) {
        config.setUseAggressiveSubstitutions(useAggressiveSubstitutions);
    }

    /**
     * Use this method to specify to use new boolean field ergonomics for generated API
     *
     * Defaults to false to be backwards compatible
     *
     * @param useBooleanFieldErgonomics true if applied.
     */
    public void setUseBooleanFieldErgonomics(boolean useBooleanFieldErgonomics) {
        config.setUseBooleanFieldErgonomics(useBooleanFieldErgonomics);
    }

    /**
     * Use this method to specify to use sub packages in generated code instead of single package
     *
     * Defaults to false to be backwards compatible
     *
     * @param usePackageGrouping true if applied.
     */
    public void setUsePackageGrouping(boolean usePackageGrouping) {
        config.setUsePackageGrouping(usePackageGrouping );
    }

    /**
     * Use this method to specify to only generate PrimaryKeyIndex for Types that has PrimaryKey defined
     *
     * Defaults to false to be backwards compatible
     *
     * @param reservePrimaryKeyIndexForTypeWithPrimaryKey true if applied.
     */
    public void reservePrimaryKeyIndexForTypeWithPrimaryKey(boolean reservePrimaryKeyIndexForTypeWithPrimaryKey) {
        config.setReservePrimaryKeyIndexForTypeWithPrimaryKey(reservePrimaryKeyIndexForTypeWithPrimaryKey);
    }

    /**
     * Use this method to specify to use Hollow Primitive Types instead of generating them per project
     *
     * Defaults to false to be backwards compatible
     *
     * @param useHollowPrimitiveTypes true if applied.
     */
    public void setUseHollowPrimitiveTypes(boolean useHollowPrimitiveTypes) {
        config.setUseHollowPrimitiveTypes(useHollowPrimitiveTypes);
    }

    /**
     * If setRestrictApiToFieldType is true, api code only generates {@code get<FieldName>} with return type as per schema
     *
     * Defaults to false to be backwards compatible
     *
     * @param restrictApiToFieldType true if applied.
     */
    public void setRestrictApiToFieldType(boolean restrictApiToFieldType) {
        config.setRestrictApiToFieldType(restrictApiToFieldType);
    }

    /**
     * Generate all files under {@code destinationPath}
     *
     * @throws IOException if the files cannot be generated
     */
    public void generateSourceFiles() throws IOException {
        generateFiles(destinationPath.toFile());
    }

    /**
     * Generate files under the specified directory
     *
     * @param directory the directory under which to generate files
     * @throws IOException if the files cannot be generated
     * @deprecated construct {@code HollowAPIGenerator} with a {@code destinationPath} then call {@link #generateSourceFiles()}
     */
    public void generateFiles(String directory) throws IOException {
        generateFiles(new File(directory));
    }

    /**
     * Generate files under the specified directory
     *
     * @param directory the directory under which to generate files
     * @throws IOException if the files cannot be generated
     * @deprecated construct {@code HollowAPIGenerator} with a {@code destinationPath} then call {@link #generateSourceFiles()}
     */
    public void generateFiles(File directory) throws IOException {
        if (packageName != null && !packageName.trim().isEmpty()) {
            String packageDir = packageName.replace(".", File.separator);
            if (!directory.getAbsolutePath().endsWith(packageDir)) {
                directory = new File(directory, packageDir);
            }
        }
        directory.mkdirs();

        HollowAPIClassJavaGenerator apiClassGenerator = new HollowAPIClassJavaGenerator(packageName, apiClassname,
                dataset, parameterizeClassNames, config);
        HollowAPIFactoryJavaGenerator apiFactoryGenerator = new HollowAPIFactoryJavaGenerator(packageName,
                apiClassname, dataset, config);

        HollowHashIndexGenerator hashIndexGenerator = new HollowHashIndexGenerator(packageName, apiClassname, dataset, config);

        generateFile(directory, apiClassGenerator);
        generateFile(directory, apiFactoryGenerator);
        generateFile(directory, hashIndexGenerator);

        generateFilesForHollowSchemas(directory);
    }

    /**
     * Generate files based on dataset schemas under {@code destinationPath}
     *
     * @throws IOException if the files cannot be generated
     */
    protected void generateSourceFilesForHollowSchemas() throws IOException {
        this.generateFilesForHollowSchemas(destinationPath.toFile());
    }

    /**
     * Generate files based on dataset schemas under the specified directory
     *
     * @param directory the directory under which to generate files
     * @throws IOException if the files cannot be generated
     * @deprecated construct {@code HollowAPIGenerator} with a {@code destinationPath} then call {@link #generateSourceFilesForHollowSchemas()}
     */
    protected void generateFilesForHollowSchemas(File directory) throws IOException {
        for(HollowSchema schema : dataset.getSchemas()) {
            String type = schema.getName();
            if (config.isUseHollowPrimitiveTypes() && HollowCodeGenerationUtils.isPrimitiveType(type)) continue; // skip if using hollow primitive type

            generateFile(directory, getStaticAPIGenerator(schema));
            generateFile(directory, getHollowObjectGenerator(schema));
            generateFile(directory, getHollowFactoryGenerator(schema));

            if(schema.getSchemaType() == SchemaType.OBJECT) {
                HollowObjectSchema objSchema = (HollowObjectSchema)schema;
                generateFile(directory, new HollowObjectDelegateInterfaceGenerator(packageName, objSchema,
                        ergonomicShortcuts, dataset, config));
                generateFile(directory, new HollowObjectDelegateCachedImplGenerator(packageName, objSchema,
                        ergonomicShortcuts, dataset, config));
                generateFile(directory, new HollowObjectDelegateLookupImplGenerator(packageName, objSchema,
                        ergonomicShortcuts, dataset, config));

                generateFile(directory, new HollowDataAccessorGenerator(packageName, apiClassname, objSchema,
                        dataset, config));
                if (!config.isReservePrimaryKeyIndexForTypeWithPrimaryKey()) {
                    generateFile(directory, new LegacyHollowPrimaryKeyIndexGenerator(packageName, apiClassname,
                            objSchema, dataset, config));
                } else if ((objSchema).getPrimaryKey() != null) {
                    generateFile(directory, new HollowPrimaryKeyIndexGenerator(dataset, packageName, apiClassname,
                            objSchema, config));
                    generateFile(directory, new HollowUniqueKeyIndexGenerator(packageName, apiClassname, objSchema,
                            dataset, config));
                }
            }
        }
    }

    protected void generateSourceFile(HollowJavaFileGenerator generator) throws IOException {
        this.generateFile(destinationPath.toFile(), generator);
    }

    /**
     * @param directory the directory under which to generate a file
     * @param generator the file generator
     * @throws IOException if the file cannot be generated
     * @deprecated construct {@code HollowAPIGenerator} with a {@code destinationPath} then call {@link #generateSourceFile(HollowJavaFileGenerator)}
     */
    protected void generateFile(File directory, HollowJavaFileGenerator generator) throws IOException {
        // create sub folder if not using default package and sub packages are enabled
        if ((packageName!=null && !packageName.trim().isEmpty()) && config.isUsePackageGrouping() && (generator instanceof HollowConsumerJavaFileGenerator)) {
            HollowConsumerJavaFileGenerator consumerCodeGenerator = (HollowConsumerJavaFileGenerator)generator;
            if (hasCollectionsInDataSet) consumerCodeGenerator.useCollectionsImport();
            directory = new File(directory, consumerCodeGenerator.getSubPackageName());
        }
        if (!directory.exists()) directory.mkdirs();

        FileWriter writer = new FileWriter(new File(directory, generator.getClassName() + ".java"));
        writer.write(generator.generate());
        writer.close();
    }

    protected HollowJavaFileGenerator getStaticAPIGenerator(HollowSchema schema) {
        if(schema instanceof HollowObjectSchema) {
            return new TypeAPIObjectJavaGenerator(apiClassname, packageName, (HollowObjectSchema) schema, dataset, config);
        } else if(schema instanceof HollowListSchema) {
            return new TypeAPIListJavaGenerator(apiClassname, packageName, (HollowListSchema)schema, dataset, config);
        } else if(schema instanceof HollowSetSchema) {
            return new TypeAPISetJavaGenerator(apiClassname, packageName, (HollowSetSchema)schema, dataset, config);
        } else if(schema instanceof HollowMapSchema) {
            return new TypeAPIMapJavaGenerator(apiClassname, packageName, (HollowMapSchema)schema, dataset, config);
        }

        throw new UnsupportedOperationException("What kind of schema is a " + schema.getClass().getName() + "?");
    }

    protected HollowJavaFileGenerator getHollowObjectGenerator(HollowSchema schema) {
        if(schema instanceof HollowObjectSchema) {
            return new HollowObjectJavaGenerator(packageName, apiClassname, (HollowObjectSchema) schema,
                    parameterizedTypes, parameterizeClassNames, ergonomicShortcuts, dataset, config);
        } else if(schema instanceof HollowListSchema) {
            return new HollowListJavaGenerator(packageName, apiClassname, (HollowListSchema) schema,
                    parameterizedTypes, parameterizeClassNames, dataset, config);
        } else if(schema instanceof HollowSetSchema) {
            return new HollowSetJavaGenerator(packageName, apiClassname, (HollowSetSchema) schema,
                    parameterizedTypes, parameterizeClassNames, dataset, config);
        } else if(schema instanceof HollowMapSchema) {
            return new HollowMapJavaGenerator(packageName, apiClassname, (HollowMapSchema) schema, dataset, parameterizedTypes, parameterizeClassNames, config);
        }

        throw new UnsupportedOperationException("What kind of schema is a " + schema.getClass().getName() + "?");
    }

    protected HollowFactoryJavaGenerator getHollowFactoryGenerator(HollowSchema schema) {
        return new HollowFactoryJavaGenerator(packageName, schema, dataset, config);
    }

    public static class Builder extends AbstractHollowAPIGeneratorBuilder<Builder, HollowAPIGenerator> {
        @Override
        protected HollowAPIGenerator instantiateGenerator() {
            return new HollowAPIGenerator(apiClassname, packageName, dataset, parameterizedTypes, parameterizeAllClassNames, useErgonomicShortcuts, destinationPath);
        }

        @Override
        protected Builder getBuilder() {
            return this;
        }
    }
}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.index.key;

import com.netflix.hollow.core.read.HollowReadFieldUtils;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;
import java.util.Arrays;

/**
 * Used to retrieve and test equality of PrimaryKey values for records.
 */
public class HollowPrimaryKeyValueDeriver {

    private final HollowObjectTypeReadState typeState;
    private final int[][] fieldPathIndexes;
    private final FieldType[] fieldTypes;

    /**
     * Create a new deriver.
     *
     * @param primaryKey The primary key spec
     * @param stateEngine The state engine to retrieve data from
     */
    public HollowPrimaryKeyValueDeriver(PrimaryKey primaryKey, HollowReadStateEngine stateEngine) {
        this.fieldPathIndexes = new int[primaryKey.numFields()][];
        this.fieldTypes = new FieldType[primaryKey.numFields()];

        for(int i=0;i<primaryKey.numFields();i++) {
            fieldPathIndexes[i] = primaryKey.getFieldPathIndex(stateEngine, i);
            fieldTypes[i] = primaryKey.getFieldType(stateEngine, i);
        }

        this.typeState = (HollowObjectTypeReadState) stateEngine.getTypeState(primaryKey.getType());
    }

    public HollowPrimaryKeyValueDeriver(HollowObjectTypeReadState typeState, int[][] fieldPathIndexes, FieldType[] fieldTypes) {
        this.typeState = typeState;
        this.fieldPathIndexes = fieldPathIndexes;
        this.fieldTypes = fieldTypes;
    }

    /**
     * Determine whether or not the specified ordinal contains the provided primary key value.
     *
     * @param ordinal the oridinal
     * @param keys the primary keys
     * @return true if the ordinal contains the primary keys
     */
    public boolean keyMatches(int ordinal, Object... keys) {
        if(keys.length != fieldPathIndexes.length)
            return false;

        for(int i=0;i<keys.length;i++) {
            if(!keyMatches(keys[i], ordinal, i))
                return false;
        }

        return true;
    }

    public boolean keyMatches(Object key, int ordinal, int fieldIdx) {
        HollowObjectTypeReadState typeState = this.typeState;
        HollowObjectSchema schema = typeState.getSchema();

        int lastFieldPath = fieldPathIndexes[fieldIdx].length - 1;
        for(int i=0;i<lastFieldPath;i++) {
            int fieldPosition = fieldPathIndexes[fieldIdx][i];
            ordinal = typeState.readOrdinal(ordinal, fieldPosition);
            typeState = (HollowObjectTypeReadState) schema.getReferencedTypeState(fieldPosition);
            schema = typeState.getSchema();
        }

        int lastFieldIdx = fieldPathIndexes[fieldIdx][lastFieldPath];

        switch(fieldTypes[fieldIdx]) {
            case BOOLEAN:
                Boolean b = typeState.readBoolean(ordinal, lastFieldIdx);
                if(b == key)
                    return true;
                if(b == null || key == null)
                    return false;
                return b.booleanValue() == ((Boolean)key).booleanValue();
            case BYTES:
                return Arrays.equals(typeState.readBytes(ordinal, lastFieldIdx), (byte[])key);
            case DOUBLE:
                return typeState.readDouble(ordinal, lastFieldIdx) == ((Double)key).doubleValue();
            case FLOAT:
                return typeState.readFloat(ordinal, lastFieldIdx) == ((Float)key).floatValue();
            case INT:
                return typeState.readInt(ordinal, lastFieldIdx) == ((Integer)key).intValue();
            case LONG:
                return typeState.readLong(ordinal, lastFieldIdx) == ((Long)key).longValue();
            case REFERENCE:
                return typeState.readOrdinal(ordinal, lastFieldIdx) == ((Integer)key).intValue();
            case STRING:
                return typeState.isStringFieldEqual(ordinal, lastFieldIdx, (String)key);
        }

        throw new IllegalArgumentException("I don't know how to compare a " + fieldTypes[fieldIdx]);
    }

    /**
     * Retrieve the primary key value for the specified ordinal.
     *
     * @param ordinal the oridinal
     * @return the primary keys
     */
    public Object[] getRecordKey(int ordinal) {
        Object[] results = new Object[fieldPathIndexes.length];

        for (int i = 0; i < fieldPathIndexes.length; i++) {
            results[i] = readValue(ordinal, i);
        }
        return results;
    }

    private Object readValue(int ordinal, int fieldIdx) {
        HollowObjectTypeReadState typeState = this.typeState;
        HollowObjectSchema schema = typeState.getSchema();

        int lastFieldPath = fieldPathIndexes[fieldIdx].length - 1;
        for (int i = 0; i < lastFieldPath; i++) {
            int fieldPosition = fieldPathIndexes[fieldIdx][i];
            ordinal = typeState.readOrdinal(ordinal, fieldPosition);
            typeState = (HollowObjectTypeReadState) schema.getReferencedTypeState(fieldPosition);
            schema = typeState.getSchema();
        }

        return HollowReadFieldUtils.fieldValueObject(typeState, ordinal, fieldPathIndexes[fieldIdx][lastFieldPath]);
    }

    public int[][] getFieldPathIndexes() {
        return fieldPathIndexes;
    }

    public FieldType[] getFieldTypes() {
        return fieldTypes;
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.api.producer;

import static java.lang.System.currentTimeMillis;
import static java.util.stream.Collectors.toList;

import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.metrics.HollowMetricsCollector;
import com.netflix.hollow.api.metrics.HollowProducerMetrics;
import com.netflix.hollow.api.producer.enforcer.BasicSingleProducerEnforcer;
import com.netflix.hollow.api.producer.enforcer.SingleProducerEnforcer;
import com.netflix.hollow.api.producer.fs.HollowFilesystemBlobStager;
import com.netflix.hollow.api.producer.listener.CycleListener;
import com.netflix.hollow.api.producer.listener.HollowProducerEventListener;
import com.netflix.hollow.api.producer.validation.ValidationResult;
import com.netflix.hollow.api.producer.validation.ValidationStatus;
import com.netflix.hollow.api.producer.validation.ValidationStatusException;
import com.netflix.hollow.api.producer.validation.ValidatorListener;
import com.netflix.hollow.core.HollowConstants;
import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader;
import com.netflix.hollow.core.read.engine.HollowBlobReader;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.util.HollowObjectHashCodeFinder;
import com.netflix.hollow.core.util.HollowWriteStateCreator;
import com.netflix.hollow.core.write.HollowBlobWriter;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper;
import com.netflix.hollow.core.write.objectmapper.RecordPrimaryKey;
import com.netflix.hollow.tools.checksum.HollowChecksum;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.function.BiConsumer;
import java.util.logging.Level;
import java.util.logging.Logger;

abstract class AbstractHollowProducer {

    static final long DEFAULT_TARGET_MAX_TYPE_SHARD_SIZE = 16L * 1024L * 1024L;

    final Logger log = Logger.getLogger(AbstractHollowProducer.class.getName());
    final HollowProducer.BlobStager blobStager;
    final HollowProducer.Publisher publisher;
    final HollowProducer.Announcer announcer;
    final HollowProducer.BlobStorageCleaner blobStorageCleaner;
    HollowObjectMapper objectMapper;
    final HollowProducer.VersionMinter versionMinter;
    final ListenerSupport listeners;
    ReadStateHelper readStates;
    final Executor snapshotPublishExecutor;
    final int numStatesBetweenSnapshots;
    int numStatesUntilNextSnapshot;
    HollowProducerMetrics metrics;
    HollowMetricsCollector<HollowProducerMetrics> metricsCollector;
    final SingleProducerEnforcer singleProducerEnforcer;
    long lastSuccessfulCycle = 0;
    final HollowObjectHashCodeFinder hashCodeFinder;

    boolean isInitialized;

    @Deprecated
    public AbstractHollowProducer(
            HollowProducer.Publisher publisher,
            HollowProducer.Announcer announcer) {
        this(new HollowFilesystemBlobStager(), publisher, announcer,
                Collections.emptyList(),
                new VersionMinterWithCounter(), null, 0,
                DEFAULT_TARGET_MAX_TYPE_SHARD_SIZE, null,
                new DummyBlobStorageCleaner(), new BasicSingleProducerEnforcer(),
                null);
    }

    // The only constructor should be that which accepts a builder
    // This ensures that if the builder modified to include new state that
    // extended builders will not require modification to pass on that new state
    AbstractHollowProducer(HollowProducer.Builder<?> b) {
        this(b.stager, b.publisher, b.announcer,
                b.eventListeners,
                b.versionMinter, b.snapshotPublishExecutor,
                b.numStatesBetweenSnapshots, b.targetMaxTypeShardSize,
                b.metricsCollector, b.blobStorageCleaner, b.singleProducerEnforcer,
                b.hashCodeFinder);
    }

    private AbstractHollowProducer(
            HollowProducer.BlobStager blobStager,
            HollowProducer.Publisher publisher,
            HollowProducer.Announcer announcer,
            List<? extends HollowProducerEventListener> eventListeners,
            HollowProducer.VersionMinter versionMinter,
            Executor snapshotPublishExecutor,
            int numStatesBetweenSnapshots,
            long targetMaxTypeShardSize,
            HollowMetricsCollector<HollowProducerMetrics> metricsCollector,
            HollowProducer.BlobStorageCleaner blobStorageCleaner,
            SingleProducerEnforcer singleProducerEnforcer,
            HollowObjectHashCodeFinder hashCodeFinder) {
        this.publisher = publisher;
        this.announcer = announcer;
        this.versionMinter = versionMinter;
        this.blobStager = blobStager;
        this.singleProducerEnforcer = singleProducerEnforcer;
        this.snapshotPublishExecutor = snapshotPublishExecutor;
        this.numStatesBetweenSnapshots = numStatesBetweenSnapshots;
        this.hashCodeFinder = hashCodeFinder;

        HollowWriteStateEngine writeEngine = hashCodeFinder == null
                ? new HollowWriteStateEngine()
                : new HollowWriteStateEngine(hashCodeFinder);
        writeEngine.setTargetMaxTypeShardSize(targetMaxTypeShardSize);

        this.objectMapper = new HollowObjectMapper(writeEngine);
        if (hashCodeFinder != null) {
            objectMapper.doNotUseDefaultHashKeys();
        }
        this.readStates = ReadStateHelper.newDeltaChain();
        this.blobStorageCleaner = blobStorageCleaner;

        this.listeners = new ListenerSupport(eventListeners.stream().distinct().collect(toList()));

        this.metrics = new HollowProducerMetrics();
        this.metricsCollector = metricsCollector;
    }

    /**
     * @return the metrics for this producer
     */
    public HollowProducerMetrics getMetrics() {
        return this.metrics;
    }

    /**
     * Initializes the data model for the given classes.
     *

     * Data model initialization is required prior to {@link #restore(long, HollowConsumer.BlobRetriever) restoring}
     * the producer.
     * This ensures that restoration can correctly compare the producer's current data model
     * with the data model of the restored data state and manage any differences in those models
     * (such as not restoring state for any types in the restoring data model not present in the
     * producer's current data model).
     *

     * After initialization a data model initialization event will be emitted
     * to all registered data model initialization listeners
     * {@link com.netflix.hollow.api.producer.listener.DataModelInitializationListener listeners}.
     *
     * @param classes the data model classes
     * @throws IllegalArgumentException if {@code classes} is empty
     * @see #restore(long, HollowConsumer.BlobRetriever)
     */
    public void initializeDataModel(Class<?>... classes) {
        Objects.requireNonNull(classes);
        if (classes.length == 0) {
            throw new IllegalArgumentException("classes is empty");
        }

        long start = currentTimeMillis();
        for (Class<?> c : classes) {
            objectMapper.initializeTypeState(c);
        }
        listeners.listeners().fireProducerInit(currentTimeMillis() - start);

        isInitialized = true;
    }

    /**
     * Initializes the producer data model for the given schemas.
     *

     * Data model initialization is required prior to {@link #restore(long, HollowConsumer.BlobRetriever) restoring}
     * the producer.
     * This ensures that restoration can correctly compare the producer's current data model
     * with the data model of the restored data state and manage any differences in those models
     * (such as not restoring state for any types in the restoring data model not present in the
     * producer's current data model).
     *

     * After initialization a data model initialization event will be emitted
     * to all registered data model initialization listeners
     * {@link com.netflix.hollow.api.producer.listener.DataModelInitializationListener listeners}.
     *
     * @param schemas the data model classes
     * @throws IllegalArgumentException if {@code schemas} is empty
     * @see #restore(long, HollowConsumer.BlobRetriever)
     */
    public void initializeDataModel(HollowSchema... schemas) {
        Objects.requireNonNull(schemas);
        if (schemas.length == 0) {
            throw new IllegalArgumentException("classes is empty");
        }

        long start = currentTimeMillis();
        HollowWriteStateCreator.populateStateEngineWithTypeWriteStates(getWriteEngine(), Arrays.asList(schemas));
        listeners.listeners().fireProducerInit(currentTimeMillis() - start);

        isInitialized = true;
    }

    /**
     * Restores the data state to a desired version.
     *

     * Data model {@link #initializeDataModel(Class[]) initialization} is required prior to
     * restoring the producer.  This ensures that restoration can correctly compare the producer's
     * current data model with the data model of the restored data state and manage any differences
     * in those models (such as not restoring state for any types in the restoring data model not
     * present in the producer's current data model)
     *
     * @param versionDesired the desired version
     * @param blobRetriever the blob retriever
     * @return the read state of the restored state
     * @throws IllegalStateException if the producer's data model has not been initialized
     * @see #initializeDataModel(Class[])
     */
    public HollowProducer.ReadState restore(long versionDesired, HollowConsumer.BlobRetriever blobRetriever) {
        return restore(versionDesired, blobRetriever,
                (restoreFrom, restoreTo) -> restoreTo.restoreFrom(restoreFrom));
    }

    HollowProducer.ReadState hardRestore(long versionDesired, HollowConsumer.BlobRetriever blobRetriever) {
        return restore(versionDesired, blobRetriever,
                (restoreFrom, restoreTo) -> HollowWriteStateCreator.
                        populateUsingReadEngine(restoreTo, restoreFrom, false));
    }

    private HollowProducer.ReadState restore(
            long versionDesired, HollowConsumer.BlobRetriever blobRetriever,
            BiConsumer<HollowReadStateEngine, HollowWriteStateEngine> restoreAction) {
        Objects.requireNonNull(blobRetriever);
        Objects.requireNonNull(restoreAction);

        if (!isInitialized) {
            throw new IllegalStateException(
                    "You must initialize the data model of a HollowProducer with producer.initializeDataModel(...) prior to restoring");
        }

        HollowProducer.ReadState readState = null;
        ListenerSupport.Listeners localListeners = listeners.listeners();
        Status.RestoreStageBuilder status = localListeners.fireProducerRestoreStart(versionDesired);
        try {
            if (versionDesired != HollowConstants.VERSION_NONE) {
                HollowConsumer client = HollowConsumer.withBlobRetriever(blobRetriever).build();
                client.triggerRefreshTo(versionDesired);
                if (client.getCurrentVersionId() == versionDesired) {
                    readState = ReadStateHelper.newReadState(client.getCurrentVersionId(), client.getStateEngine());
                    readStates = ReadStateHelper.restored(readState);

                    // Need to restore data to new ObjectMapper since can't restore to non empty Write State Engine
                    Collection<HollowSchema> schemas = objectMapper.getStateEngine().getSchemas();
                    HollowWriteStateEngine writeEngine = hashCodeFinder == null
                            ? new HollowWriteStateEngine()
                            : new HollowWriteStateEngine(hashCodeFinder);
                    HollowWriteStateCreator.populateStateEngineWithTypeWriteStates(writeEngine, schemas);
                    HollowObjectMapper newObjectMapper = new HollowObjectMapper(writeEngine);
                    if (hashCodeFinder != null) {
                        newObjectMapper.doNotUseDefaultHashKeys();
                    }

                    restoreAction.accept(readStates.current().getStateEngine(), writeEngine);

                    status.versions(versionDesired, readState.getVersion())
                            .success();
                    objectMapper = newObjectMapper; // Restore completed successfully so swap
                } else {
                    status.versions(versionDesired, client.getCurrentVersionId());
                    throw new IllegalStateException(
                            "Unable to reach requested version to restore from: " + versionDesired);
                }
            }
        } catch (Throwable th) {
            status.fail(th);
            throw th;
        } finally {
            localListeners.fireProducerRestoreComplete(status);
        }
        return readState;
    }

    public HollowWriteStateEngine getWriteEngine() {
        return objectMapper.getStateEngine();
    }

    public HollowObjectMapper getObjectMapper() {
        return objectMapper;
    }

    /**
     * Invoke this method to alter runCycle behavior. If this Producer is not primary, runCycle is a no-op. Note that by default,
     * SingleProducerEnforcer is instantiated as BasicSingleProducerEnforcer, which is initialized to return true for isPrimary()
     *
     * @param doEnable true if enable primary producer, if false
     * @return true if the intended action was successful
     */
    public boolean enablePrimaryProducer(boolean doEnable) {
        if (doEnable) {
            singleProducerEnforcer.enable();
        } else {
            singleProducerEnforcer.disable();
        }
        return (singleProducerEnforcer.isPrimary() == doEnable);
    }

    long runCycle(HollowProducer.Incremental.IncrementalPopulator incrementalPopulator, HollowProducer.Populator populator) {
        ListenerSupport.Listeners localListeners = listeners.listeners();

        if (!singleProducerEnforcer.isPrimary()) {
            // TODO: minimum time spacing between cycles
            log.log(Level.INFO, "cycle not executed -- not primary");
            localListeners.fireCycleSkipped(CycleListener.CycleSkipReason.NOT_PRIMARY_PRODUCER);
            return lastSuccessfulCycle;
        }

        long toVersion = versionMinter.mint();

        if (!readStates.hasCurrent()) {
            localListeners.fireNewDeltaChain(toVersion);
        }

        Status.StageWithStateBuilder cycleStatus = localListeners.fireCycleStart(toVersion);
        try {
            return runCycle(localListeners, incrementalPopulator, populator, cycleStatus, toVersion);
        } finally {
            localListeners.fireCycleComplete(cycleStatus);
            metrics.updateCycleMetrics(cycleStatus.build(), cycleStatus.readState, cycleStatus.version);
            if (metricsCollector != null) {
                metricsCollector.collect(metrics);
            }
        }
    }

    long runCycle(
            ListenerSupport.Listeners listeners,
            HollowProducer.Incremental.IncrementalPopulator incrementalPopulator, HollowProducer.Populator populator,
            Status.StageWithStateBuilder cycleStatus, long toVersion) {
        // 1. Begin a new cycle
        Artifacts artifacts = new Artifacts();
        HollowWriteStateEngine writeEngine = getWriteEngine();
        try {
            // 1a. Prepare the write state
            writeEngine.prepareForNextCycle();

            // 2. Populate the state
            populate(listeners, incrementalPopulator, populator, toVersion);

            // 3. Produce a new state if there's work to do
            if (writeEngine.hasChangedSinceLastCycle()) {
                // 3a. Publish, run checks & validation, then announce new state consumers
                publish(listeners, toVersion, artifacts);

                ReadStateHelper candidate = readStates.roundtrip(toVersion);
                cycleStatus.readState(candidate.pending());
                candidate = checkIntegrity(listeners, candidate, artifacts);

                try {
                    validate(listeners, candidate.pending());

                    announce(listeners, candidate.pending());

                    readStates = candidate.commit();
                    cycleStatus.readState(readStates.current()).success();
                } catch (Throwable th) {
                    if (artifacts.hasReverseDelta()) {
                        applyDelta(artifacts.reverseDelta, candidate.pending().getStateEngine());
                        readStates = candidate.rollback();
                    }
                    throw th;
                }
                lastSuccessfulCycle = toVersion;
            } else {
                // 3b. Nothing to do; reset the effects of Step 2
                // Return the lastSucessfulCycle to the caller thereby
                // the callee can track that version against consumers
                // without having to listen to events.
                // Consistently report the version that would be used if
                // data had been published for the events.  This
                // is for consistency in tracking
                writeEngine.resetToLastPrepareForNextCycle();
                cycleStatus.success();
                listeners.fireNoDelta(toVersion);
            }
        } catch (Throwable th) {
            writeEngine.resetToLastPrepareForNextCycle();
            cycleStatus.fail(th);

            if (th instanceof RuntimeException) {
                throw (RuntimeException) th;
            }
            throw new RuntimeException(th);
        } finally {
            artifacts.cleanup();
        }
        return lastSuccessfulCycle;
    }

    /**
     * Adds a listener to this producer.
     *

     * If the listener was previously added to this consumer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is added, concurrently, during the occurrence of a cycle or restore then the listener will not
     * receive events until the next cycle or restore.  The listener may also be removed concurrently.
     *
     * @param listener the listener to add
     */
    public void addListener(HollowProducerListener listener) {
        listeners.addListener(listener);
    }

    /**
     * Adds a listener to this producer.
     *

     * If the listener was previously added to this consumer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is added, concurrently, during the occurrence of a cycle or restore then the listener will not
     * receive events until the next cycle or restore.  The listener may also be removed concurrently.
     *
     * @param listener the listener to add
     */
    public void addListener(HollowProducerEventListener listener) {
        listeners.addListener(listener);
    }

    /**
     * Removes a listener to this producer.
     *

     * If the listener was not previously added to this producer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is removed, concurrently, during the occurrence of a cycle or restore then the listener will
     * receive all events for that cycle or restore but not receive events for a subsequent cycle or restore.
     *
     * @param listener the listener to remove
     */
    public void removeListener(HollowProducerListener listener) {
        listeners.removeListener(listener);
    }

    /**
     * Removes a listener to this producer.
     *

     * If the listener was not previously added to this producer, as determined by reference equality or {@code Object}
     * equality, then this method does nothing.
     *

     * If a listener is removed, concurrently, during the occurrence of a cycle or restore then the listener will
     * receive all events for that cycle or restore but not receive events for a subsequent cycle or restore.
     *
     * @param listener the listener to remove
     */
    public void removeListener(HollowProducerEventListener listener) {
        listeners.removeListener(listener);
    }

    void populate(
            ListenerSupport.Listeners listeners,
            HollowProducer.Incremental.IncrementalPopulator incrementalPopulator, HollowProducer.Populator populator,
            long toVersion) throws Exception {
        assert incrementalPopulator != null ^ populator != null;

        Status.StageBuilder populateStatus = listeners.firePopulateStart(toVersion);
        try {
            if (incrementalPopulator != null) {
                // Incremental population is a sub-stage of the population stage
                // This ensures good integration with existing population listeners if this sub-stage fails
                // then the population stage will fail
                populator = incrementalPopulate(listeners, incrementalPopulator, toVersion);
            }

            try (CloseableWriteState writeState = new CloseableWriteState(toVersion, objectMapper,
                    readStates.current())) {
                populator.populate(writeState);
                populateStatus.success();
            }
        } catch (Throwable th) {
            populateStatus.fail(th);
            throw th;
        } finally {
            listeners.firePopulateComplete(populateStatus);
        }
    }

    HollowProducer.Populator incrementalPopulate(
            ListenerSupport.Listeners listeners,
            HollowProducer.Incremental.IncrementalPopulator incrementalPopulator,
            long toVersion) throws Exception {
        ConcurrentHashMap<RecordPrimaryKey, Object> events = new ConcurrentHashMap<>();
        Status.IncrementalPopulateBuilder incrementalPopulateStatus = listeners.fireIncrementalPopulateStart(toVersion);
        try (CloseableIncrementalWriteState iws = new CloseableIncrementalWriteState(events, getObjectMapper())) {
            incrementalPopulator.populate(iws);
            incrementalPopulateStatus.success();

            long removed = events.values().stream()
                    .filter(o -> o == HollowIncrementalCyclePopulator.DELETE_RECORD).count();
            long addedOrModified = events.size() - removed;
            incrementalPopulateStatus.changes(removed, addedOrModified);
        } catch (Throwable th) {
            incrementalPopulateStatus.fail(th);
            throw th;
        } finally {
            listeners.fireIncrementalPopulateComplete(incrementalPopulateStatus);
        }

        return new HollowIncrementalCyclePopulator(events, 1.0);
    }

    /*
     * Publish the write state, storing the artifacts in the provided object. Visible for testing.
     */
    void publish(ListenerSupport.Listeners listeners, long toVersion, Artifacts artifacts) throws IOException {
        Status.StageBuilder psb = listeners.firePublishStart(toVersion);
        try {
            artifacts.snapshot = stageBlob(listeners,
                    blobStager.openSnapshot(toVersion));

            if (readStates.hasCurrent()) {
                artifacts.delta = stageBlob(listeners,
                        blobStager.openDelta(readStates.current().getVersion(), toVersion));
                artifacts.reverseDelta = stageBlob(listeners,
                        blobStager.openReverseDelta(toVersion, readStates.current().getVersion()));

                publishBlob(listeners, artifacts.delta);
                publishBlob(listeners, artifacts.reverseDelta);

                if (--numStatesUntilNextSnapshot < 0) {
                    if (snapshotPublishExecutor == null) {
                        publishBlob(listeners, artifacts.snapshot);
                        artifacts.markSnapshotPublishComplete();
                    } else {
                        // Submit the publish blob task to the executor
                        publishSnapshotBlobAsync(listeners, artifacts);
                    }
                    numStatesUntilNextSnapshot = numStatesBetweenSnapshots;
                } else {
                    artifacts.markSnapshotPublishComplete();
                }
            } else {
                publishBlob(listeners, artifacts.snapshot);
                artifacts.markSnapshotPublishComplete();
                numStatesUntilNextSnapshot = numStatesBetweenSnapshots;
            }

            psb.success();
        } catch (Throwable throwable) {
            psb.fail(throwable);
            throw throwable;
        } finally {
            listeners.firePublishComplete(psb);
        }
    }

    private HollowProducer.Blob stageBlob(ListenerSupport.Listeners listeners, HollowProducer.Blob blob)
            throws IOException {
        Status.PublishBuilder builder = new Status.PublishBuilder();
        HollowBlobWriter writer = new HollowBlobWriter(getWriteEngine());
        try {
            builder.blob(blob);
            blob.write(writer);
            builder.success();
            return blob;
        } catch (Throwable t) {
            builder.fail(t);
            throw t;
        } finally {
            listeners.fireBlobStage(builder);
        }
    }

    private void publishBlob(ListenerSupport.Listeners listeners, HollowProducer.Blob blob) {
        Status.PublishBuilder builder = new Status.PublishBuilder();
        try {
            builder.blob(blob);
            publishBlob(blob);
            builder.success();
        } catch (Throwable t) {
            builder.fail(t);
            throw t;
        } finally {
            listeners.fireBlobPublish(builder);
            metrics.updateBlobTypeMetrics(builder.build(), blob);
            if (metricsCollector != null) {
                metricsCollector.collect(metrics);
            }
        }
    }

    private void publishSnapshotBlobAsync(ListenerSupport.Listeners listeners, Artifacts artifacts) {
        HollowProducer.Blob blob = artifacts.snapshot;
        CompletableFuture<HollowProducer.Blob> cf = new CompletableFuture<>();
        try {
            snapshotPublishExecutor.execute(() -> {
                Status.StageBuilder builder = new Status.StageBuilder();
                try {
                    publishBlob(blob);
                    builder.success();
                    // Any dependent task that needs access to the blob contents should
                    // not execute asynchronously otherwise the blob will be cleaned up
                    cf.complete(blob);
                } catch (Throwable t) {
                    builder.fail(t);
                    cf.completeExceptionally(t);
                    throw t;
                } finally {
                    metrics.updateBlobTypeMetrics(builder.build(), blob);
                    if (metricsCollector != null) {
                        metricsCollector.collect(metrics);
                    }
                }
                artifacts.markSnapshotPublishComplete();
            });
        } catch (Throwable t) {
            cf.completeExceptionally(t);
            metrics.updateBlobTypeMetrics(new Status.StageBuilder().fail(t).build(), blob);
            if (metricsCollector != null) {
                metricsCollector.collect(metrics);
            }
            throw t;
        } finally {
            listeners.fireBlobPublishAsync(cf);
        }
    }

    private void publishBlob(HollowProducer.Blob b) {
        try {
            publisher.publish(b);
        } finally {
            blobStorageCleaner.clean(b.getType());
        }
    }

    /**
     * Given these read states
     *

     * * S(cur) at the currently announced version
     * * S(pnd) at the pending version
     *

     * Ensure that:
     *

     * S(cur).apply(forwardDelta).checksum == S(pnd).checksum
     * S(pnd).apply(reverseDelta).checksum == S(cur).checksum
     *
     * @return updated read states
     */
    private ReadStateHelper checkIntegrity(
            ListenerSupport.Listeners listeners, ReadStateHelper readStates, Artifacts artifacts) throws Exception {
        Status.StageWithStateBuilder status = listeners.fireIntegrityCheckStart(readStates.pending());
        try {
            ReadStateHelper result = readStates;
            HollowReadStateEngine pending = readStates.pending().getStateEngine();
            readSnapshot(artifacts.snapshot, pending);

            if (readStates.hasCurrent()) {
                HollowReadStateEngine current = readStates.current().getStateEngine();

                log.info("CHECKSUMS");
                HollowChecksum currentChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending);
                log.info("  CUR        " + currentChecksum);

                HollowChecksum pendingChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current);
                log.info("         PND " + pendingChecksum);

                if (artifacts.hasDelta()) {
                    if (!artifacts.hasReverseDelta()) {
                        throw new IllegalStateException("Both a delta and reverse delta are required");
                    }

                    // FIXME: timt: future cycles will fail unless both deltas validate
                    applyDelta(artifacts.delta, current);
                    HollowChecksum forwardChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending);
                    //out.format("  CUR => PND %s\n", forwardChecksum);
                    if (!forwardChecksum.equals(pendingChecksum)) {
                        throw new HollowProducer.ChecksumValidationException(HollowProducer.Blob.Type.DELTA);
                    }

                    applyDelta(artifacts.reverseDelta, pending);
                    HollowChecksum reverseChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current);
                    //out.format("  CUR <= PND %s\n", reverseChecksum);
                    if (!reverseChecksum.equals(currentChecksum)) {
                        throw new HollowProducer.ChecksumValidationException(HollowProducer.Blob.Type.REVERSE_DELTA);
                    }
                    result = readStates.swap();
                }
            }
            status.success();
            return result;
        } catch (Throwable th) {
            status.fail(th);
            throw th;
        } finally {
            listeners.fireIntegrityCheckComplete(status);
        }
    }

    private void readSnapshot(HollowProducer.Blob blob, HollowReadStateEngine stateEngine) throws IOException {
        try (InputStream is = blob.newInputStream()) {
            new HollowBlobReader(stateEngine, new HollowBlobHeaderReader()).readSnapshot(is);
        }
    }

    private void applyDelta(HollowProducer.Blob blob, HollowReadStateEngine stateEngine) throws IOException {
        try (InputStream is = blob.newInputStream()) {
            new HollowBlobReader(stateEngine, new HollowBlobHeaderReader()).applyDelta(is);
        }
    }

    private void validate(ListenerSupport.Listeners listeners, HollowProducer.ReadState readState) {
        Status.StageWithStateBuilder psb = listeners.fireValidationStart(readState);

        ValidationStatus status = null;
        try {
            // Stream over the concatenation of the old and new validators
            List<ValidationResult> results =
                    listeners.getListeners(ValidatorListener.class)
                            .map(v -> {
                                try {
                                    return v.onValidate(readState);
                                } catch (RuntimeException e) {
                                    return ValidationResult.from(v).error(e);
                                }
                            })
                            .collect(toList());

            status = new ValidationStatus(results);

            if (!status.passed()) {
                ValidationStatusException e = new ValidationStatusException(
                        status, "One or more validations failed. Please check individual failures.");
                psb.fail(e);
                throw e;
            }
            psb.success();
        } finally {
            listeners.fireValidationComplete(psb, status);
        }
    }

    private void announce(ListenerSupport.Listeners listeners, HollowProducer.ReadState readState) {
        if (announcer != null) {
            Status.StageWithStateBuilder status = listeners.fireAnnouncementStart(readState);
            try {
                announcer.announce(readState.getVersion());
                status.success();
            } catch (Throwable th) {
                status.fail(th);
                throw th;
            } finally {
                listeners.fireAnnouncementComplete(status);
            }
        }
    }

    static final class Artifacts {
        HollowProducer.Blob snapshot = null;
        HollowProducer.Blob delta = null;
        HollowProducer.Blob reverseDelta = null;

        boolean cleanupCalled;
        boolean snapshotPublishComplete;

        synchronized void cleanup() {
            cleanupCalled = true;

            cleanupSnapshot();

            if (delta != null) {
                delta.cleanup();
                delta = null;
            }
            if (reverseDelta != null) {
                reverseDelta.cleanup();
                reverseDelta = null;
            }
        }

        synchronized void markSnapshotPublishComplete() {
            snapshotPublishComplete = true;

            cleanupSnapshot();
        }

        private void cleanupSnapshot() {
            if (cleanupCalled && snapshotPublishComplete && snapshot != null) {
                snapshot.cleanup();
                snapshot = null;
            }
        }

        boolean hasDelta() {
            return delta != null;
        }

        boolean hasReverseDelta() {
            return reverseDelta != null;
        }
    }

    /**
     * This Dummy blob storage cleaner does nothing
     */
    static class DummyBlobStorageCleaner extends HollowProducer.BlobStorageCleaner {

        @Override
        public void cleanSnapshots() {
        }

        @Override
        public void cleanDeltas() {
        }

        @Override
        public void cleanReverseDeltas() {
        }
    }

}


/*
 *  Copyright 2016-2019 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.hollow.core.read.engine;

import com.netflix.hollow.api.error.SchemaNotFoundException;
import com.netflix.hollow.core.HollowStateEngine;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.memory.pool.RecyclingRecycler;
import com.netflix.hollow.core.read.dataaccess.HollowDataAccess;
import com.netflix.hollow.core.read.dataaccess.HollowTypeDataAccess;
import com.netflix.hollow.core.read.engine.map.HollowMapTypeReadState;
import com.netflix.hollow.core.read.engine.set.HollowSetTypeReadState;
import com.netflix.hollow.core.read.missing.DefaultMissingDataHandler;
import com.netflix.hollow.core.read.missing.MissingDataHandler;
import com.netflix.hollow.core.schema.HollowListSchema;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSetSchema;
import com.netflix.hollow.core.util.DefaultHashCodeFinder;
import com.netflix.hollow.core.util.HollowObjectHashCodeFinder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * A HollowReadStateEngine is our main handle to the current state of a Hollow dataset as a data consumer.
 * <p>
 * A dataset changes over time.  A core concept in Hollow is that the timeline for a changing dataset can be
 * broken down into discrete data states, each of which is a complete snapshot of the data at a particular point in time.
 * Data consumers handle data states with a HollowReadStateEngine.
 */
public class HollowReadStateEngine implements HollowStateEngine, HollowDataAccess {

    private final Map<String, HollowTypeReadState> typeStates;
    private final Map<String, List<HollowTypeStateListener>> listeners;
    private final HollowObjectHashCodeFinder hashCodeFinder;
    private final boolean listenToAllPopulatedOrdinals;
    private ArraySegmentRecycler memoryRecycler;
    private Map<String,String> headerTags;
    private Set<String> typesWithDefinedHashCodes = new HashSet<String>();

    private long currentRandomizedTag;

    private MissingDataHandler missingDataHandler = new DefaultMissingDataHandler();

    public HollowReadStateEngine() {
        this(DefaultHashCodeFinder.INSTANCE, true, new RecyclingRecycler());
    }

    public HollowReadStateEngine(boolean listenToAllPopulatedOrdinals) {
        this(DefaultHashCodeFinder.INSTANCE, listenToAllPopulatedOrdinals, new RecyclingRecycler());
    }

    public HollowReadStateEngine(ArraySegmentRecycler recycler) {
        this(DefaultHashCodeFinder.INSTANCE, true, recycler);
    }

    public HollowReadStateEngine(boolean listenToAllPopulatedOrdinals, ArraySegmentRecycler recycler) {
        this(DefaultHashCodeFinder.INSTANCE, listenToAllPopulatedOrdinals, recycler);
    }

    @Deprecated
    public HollowReadStateEngine(HollowObjectHashCodeFinder hashCodeFinder) {
        this(hashCodeFinder, true, new RecyclingRecycler());
    }

    @Deprecated
    public HollowReadStateEngine(HollowObjectHashCodeFinder hashCodeFinder, boolean listenToAllPopulatedOrdinals, ArraySegmentRecycler recycler) {
        this.typeStates = new HashMap<String, HollowTypeReadState>();
        this.listeners = new HashMap<String, List<HollowTypeStateListener>>();
        this.hashCodeFinder = hashCodeFinder;
        this.memoryRecycler = recycler;
        this.listenToAllPopulatedOrdinals = listenToAllPopulatedOrdinals;
    }

    @Override
    public HollowObjectHashCodeFinder getHashCodeFinder() {
        return hashCodeFinder;
    }

    protected void addTypeState(HollowTypeReadState typeState) {
        typeStates.put(typeState.getSchema().getName(), typeState);

        if(listenToAllPopulatedOrdinals) {
            typeState.addListener(new PopulatedOrdinalListener());
        }

        List<HollowTypeStateListener> list = listeners.get(typeState.getSchema().getName());
        if(list != null) {
            for(HollowTypeStateListener listener : list)
                typeState.addListener(listener);
        }
    }

    /**
     * Add a {@link HollowTypeStateListener} to a type.
     *
     * @param typeName the type name
     * @param listener the listener to add
     */
    public void addTypeListener(String typeName, HollowTypeStateListener listener) {
        List<HollowTypeStateListener> list = listeners.get(typeName);
        if(list == null) {
            list = new ArrayList<HollowTypeStateListener>();
            listeners.put(typeName, list);
        }

        list.add(listener);

        HollowTypeReadState typeState = typeStates.get(typeName);
        if(typeState != null)
            typeState.addListener(listener);
    }

    void wireTypeStatesToSchemas() {
        for(HollowTypeReadState state : typeStates.values()) {
            switch(state.getSchema().getSchemaType()) {
            case OBJECT:
                HollowObjectSchema objSchema = (HollowObjectSchema)state.getSchema();
                for(int i=0;i<objSchema.numFields();i++) {
                    if(objSchema.getReferencedType(i) != null)
                        objSchema.setReferencedTypeState(i, typeStates.get(objSchema.getReferencedType(i)));
                }
                break;
            case LIST:
                HollowListSchema listSchema = (HollowListSchema)state.getSchema();
                listSchema.setElementTypeState(typeStates.get(listSchema.getElementType()));
                break;
            case SET:
                HollowSetSchema setSchema = (HollowSetSchema)state.getSchema();
                setSchema.setElementTypeState(typeStates.get(setSchema.getElementType()));
                ((HollowSetTypeReadState)state).buildKeyDeriver();
                break;
            case MAP:
                HollowMapSchema mapSchema = (HollowMapSchema)state.getSchema();
                mapSchema.setKeyTypeState(typeStates.get(mapSchema.getKeyType()));
                mapSchema.setValueTypeState(typeStates.get(mapSchema.getValueType()));
                ((HollowMapTypeReadState)state).buildKeyDeriver();
                break;
            }
        }
    }

    /**
     * Calculates the data size of a read state engine which is defined as the approximate heap footprint by iterating
     * over the read state shards in each type state
     * @return the heap footprint of the read state engine
     */
    public long calcApproxDataSize() {
        return this.getAllTypes()
                .stream()
                .map(this::getTypeState)
                .mapToLong(HollowTypeReadState::getApproximateHeapFootprintInBytes)
                .sum();
    }

    @Override
    public HollowTypeDataAccess getTypeDataAccess(String type) {
        return typeStates.get(type);
    }

    @Override
    public HollowTypeDataAccess getTypeDataAccess(String type, int ordinal) {
        return typeStates.get(type);
    }

    @Override
    public Collection<String> getAllTypes() {
        return typeStates.keySet();
    }

    public HollowTypeReadState getTypeState(String type) {
        return typeStates.get(type);
    }

    public Collection<HollowTypeReadState> getTypeStates() {
        return typeStates.values();
    }

    public ArraySegmentRecycler getMemoryRecycler() {
        return memoryRecycler;
    }

    public boolean isListenToAllPopulatedOrdinals() {
        return listenToAllPopulatedOrdinals;
    }

    @Override
    public List<HollowSchema> getSchemas() {
        List<HollowSchema> schemas = new ArrayList<HollowSchema>();

        for(Map.Entry<String, HollowTypeReadState> entry : typeStates.entrySet()) {
            schemas.add(entry.getValue().getSchema());
        }

        return schemas;
    }

    @Override
    public HollowSchema getSchema(String type) {
        HollowTypeReadState typeState = getTypeState(type);
        return typeState == null ? null : typeState.getSchema();
    }

    @Override
    public HollowSchema getNonNullSchema(String type) {
        HollowSchema schema = getSchema(type);
        if (schema == null) {
            throw new SchemaNotFoundException(type, getAllTypes());
        }
        return schema;
    }

    protected void afterInitialization() { }

    public void setMissingDataHandler(MissingDataHandler handler) {
        this.missingDataHandler = handler;
    }

    @Override
    public MissingDataHandler getMissingDataHandler() {
        return missingDataHandler;
    }

    public void setHeaderTags(Map<String, String> headerTags) {
        this.headerTags = headerTags;
        populatedDefinedHashCodesTypesIfHeaderTagIsPresent();
    }

    @Override
    public Map<String, String> getHeaderTags() {
        return headerTags;
    }

    @Override
    public String getHeaderTag(String name) {
        return headerTags.get(name);
    }

    public void invalidate() {
        listeners.clear();

        for(Map.Entry<String, HollowTypeReadState> entry : typeStates.entrySet())
            entry.getValue().invalidate();

        memoryRecycler = null;
    }

    @Override
    public void resetSampling() {
        for(Map.Entry<String, HollowTypeReadState> entry : typeStates.entrySet())
            entry.getValue().getSampler().reset();
    }

    @Override
    public boolean hasSampleResults() {
        for(Map.Entry<String, HollowTypeReadState> entry : typeStates.entrySet())
            if(entry.getValue().getSampler().hasSampleResults())
                return true;
        return false;
    }

    public boolean updatedLastCycle() {
        for(Map.Entry<String, HollowTypeReadState> entry : typeStates.entrySet()) {
            if(entry.getValue().getListener(PopulatedOrdinalListener.class).updatedLastCycle())
                return true;
        }
        return false;
    }

    public Set<String> getTypesWithDefinedHashCodes() {
        return typesWithDefinedHashCodes;
    }

    public long getCurrentRandomizedTag() {
        return currentRandomizedTag;
    }

    public void setCurrentRandomizedTag(long currentRandomizedTag) {
        this.currentRandomizedTag = currentRandomizedTag;
    }

    private void populatedDefinedHashCodesTypesIfHeaderTagIsPresent() {
        String definedHashCodesTag = headerTags.get(HollowObjectHashCodeFinder.DEFINED_HASH_CODES_HEADER_NAME);
        if(definedHashCodesTag == null || "".equals(definedHashCodesTag)) {
            this.typesWithDefinedHashCodes = Collections.<String>emptySet();
        } else {
            Set<String>definedHashCodeTypes = new HashSet<String>();
            for(String type : definedHashCodesTag.split(","))
                definedHashCodeTypes.add(type);
            this.typesWithDefinedHashCodes = definedHashCodeTypes;
        }
    }

}