package org.nustaq.serialization.coders;

import com.fasterxml.jackson.core.*;
import org.nustaq.serialization.*;
import org.nustaq.serialization.util.FSTInputStream;
import org.nustaq.serialization.util.FSTUtil;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

/**
 * Created by moelrue on 5/21/15.
 */
public class FSTJsonDecoder implements FSTDecoder {

    protected FSTConfiguration conf;
    protected FSTJsonFieldNames fieldNames;

    protected JsonParser input;
    protected JsonFactory fac;

    protected FSTInputStream fstInput;
    protected String unknownFallbackReadFieldName; // contains read fieldName in case of Unknown resulting from plain JSon structure
    protected HashMap<String,Class> clzCache = new HashMap<>(31);
    protected String lastUnknown;
    protected int unknownNestLevel;

    public FSTJsonDecoder(FSTConfiguration conf) {
        fac = conf.getCoderSpecific();
        setConf(conf);
    }

    @Override
    public void setConf(FSTConfiguration conf) {
        this.conf = conf;
        fieldNames = conf.getJsonFieldNames();
    }

    @Override
    public String readStringUTF() throws IOException {
        if ( unknownFallbackReadFieldName != null ) {
            String unkReadAhead = unknownFallbackReadFieldName;
            unknownFallbackReadFieldName = null;
            return unkReadAhead;
        }
        JsonToken jsonToken = input.nextToken();
        if ( jsonToken == JsonToken.VALUE_NULL )
            return null;
        if ( jsonToken == JsonToken.FIELD_NAME )
            return input.getCurrentName();
        return input.getText();
    }

    @Override
    public String readStringAsc() throws IOException {
        return (String) input.nextTextValue();
    }

    @Override
    /**
     * if array is null => create own array. if len == -1 => use len read
     */
    public Object readFPrimitiveArray(Object array, Class componentType, int len) {
        try {
            if (componentType == int.class) {
                int[] da = (int[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = (int) input.getIntValue();
                }
                return da;
            }
            else if (componentType == long.class) {
                long[] da = (long[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = input.getLongValue();
                }
                return da;
            } else if (componentType == double.class) {
                double[] da = (double[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken();
                    da[i] = input.getDoubleValue();
                }
                return da;
            } else if (componentType == float.class) {
                float[] da = (float[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken();
                    da[i] = input.getFloatValue();
                }
                return da;
            } else if (componentType == boolean.class) {
                boolean[] da = (boolean[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = input.getBooleanValue();
                }
                return da;
            }
            else if (componentType == byte.class) {
                byte[] da = (byte[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = input.getByteValue();
                }
                return da;
            }
            else if (componentType == short.class) {
                short[] da = (short[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = input.getShortValue();
                }
                return da;
            }
            else if (componentType == char.class) {
                char[] da = (char[]) array;
                for (int i = 0; i < da.length; i++) {
                    input.nextToken(); da[i] = (char) input.getIntValue();
                }
                return da;
            }
            else throw new RuntimeException("unsupported type " + componentType.getName());
        } catch (Exception e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
        return null;
    }

    @Override
    public void readFIntArr(int len, int[] arr) throws IOException {
        JsonToken jsonToken = input.nextToken();
        if ( ! jsonToken.isStructStart() )
            throw new RuntimeException("Expected array start");
        for (int i = 0; i < len; i++) {
            input.nextToken(); arr[i] = input.getIntValue();
        }
    }

    @Override
    public int readFInt() throws IOException {
        return input.nextIntValue(-1);
    }

    @Override
    public double readFDouble() throws IOException {
        input.nextToken();
        return input.getDoubleValue();
    }

    @Override
    public float readFFloat() throws IOException {
        input.nextToken();
        return input.getFloatValue();
    }

    @Override
    public byte readFByte() throws IOException {
        input.nextToken();
        int currentTokenId = input.getCurrentTokenId();
        if ( currentTokenId == JsonTokenId.ID_FALSE ) {
            return 0;
        }
        if ( currentTokenId == JsonTokenId.ID_TRUE ) {
            return 1;
        }
        return input.getByteValue();
    }

    @Override
    public int readIntByte() throws IOException {
        input.nextToken();
        return input.getByteValue();
    }

    @Override
    public long readFLong() throws IOException {
        input.nextToken();
        return input.getLongValue();
    }

    @Override
    public char readFChar() throws IOException {
        input.nextToken();
        return (char) input.getIntValue();
    }

    @Override
    public short readFShort() throws IOException {
        input.nextToken();
        return input.getShortValue();
    }

    @Override
    public int readPlainInt() throws IOException {
        throw new RuntimeException("not supported");
    }

    @Override
    public byte[] getBuffer() {
        return fstInput.buf;
    }

    @Override
    public int getInputPos() {
        JsonLocation currentLocation = input.getCurrentLocation();
        long byteOffset = currentLocation.getByteOffset();
        if ( input.getCurrentToken() == JsonToken.FIELD_NAME )
            byteOffset-=2; // eager parsing of jackson ':' + '['/'{'
        return (int) byteOffset;
    }

    @Override
    public void moveTo(int position) {
        throw new RuntimeException("not supported");
    }

    @Override
    public void setInputStream(InputStream in) {
        try {
            if ( fstInput != null )
                fstInput.initFromStream(in);
            else
                fstInput = new FSTInputStream(in);
            if ( in != FSTObjectInput.emptyStream )
                createParser();
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    @Override
    public int ensureReadAhead(int bytes) {
        return 0;
    }

    @Override
    public void reset() {
        fstInput.reset();
        input = null;
        unknownNestLevel = 0;
        firstCall = true;
    }

    @Override
    public void resetToCopyOf(byte[] bytes, int off, int len) {
        if (off != 0 )
            throw new RuntimeException("not supported");
        byte b[] = new byte[len];
        System.arraycopy(bytes,off,b,0,len);
        if ( fstInput == null )
            fstInput = new FSTInputStream(b);
        else
            fstInput.resetForReuse(bytes,len);
        try {
            createParser();
            unknownNestLevel = 0;
            firstCall = true;
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    public void createParser() throws IOException {
        if ( input != null )
            input.close();
        input = fac.createParser(fstInput);
    }

    @Override
    public void resetWith(byte[] bytes, int len) {
        if ( fstInput == null ) {
            fstInput = new FSTInputStream(bytes,0,len);
        }
        else {
            fstInput.resetForReuse(bytes,len);
        }
        try {
            createParser();
            unknownNestLevel = 0; // fixme: should delegate to core reset (not now as I don't want to break things)
            firstCall = true;
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    public int getObjectHeaderLen() // len field of last header read (if avaiable)
    {
        return lastObjectLen;
    }

    int lastObjectLen;
    Class lastDirectClass;
    boolean firstCall = true;
    public byte readObjectHeaderTag() throws IOException {
        boolean isFirst = firstCall;
        firstCall = false;
        lastObjectLen = -1;
        lastReadDirectObject = null;
        lastDirectClass = null;
        JsonToken jsonToken = input.nextToken();
        if ( jsonToken == JsonToken.END_OBJECT )
            jsonToken = input.nextToken();
        switch (jsonToken) {
            case VALUE_STRING: {
                lastReadDirectObject = input.getValueAsString();
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            case VALUE_TRUE: {
                lastReadDirectObject = Boolean.TRUE;
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            case VALUE_FALSE: {
                lastReadDirectObject = Boolean.FALSE;
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            case VALUE_NUMBER_INT: {
                lastReadDirectObject = input.getNumberValue();
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            case VALUE_NUMBER_FLOAT: {
                lastReadDirectObject = input.getDoubleValue();
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            case START_ARRAY: {
                if (unknownNestLevel > 0 || isFirst) {
                    lastReadDirectObject = createUnknownArray();
                    return FSTObjectOutput.DIRECT_ARRAY_OBJECT;
                } else {
                    lastReadDirectObject = createPrimitiveArrayFrom(readJSonArr2List(getTmpList()));
                    return FSTObjectOutput.DIRECT_ARRAY_OBJECT;
                }
            }
            case VALUE_NULL: {
                lastReadDirectObject = null;
                return FSTObjectOutput.NULL;
            }
        }
        if ( jsonToken != JsonToken.START_OBJECT ) {
            if ( jsonToken == JsonToken.END_ARRAY ) { // tweak to get end marker when reading json array with readObject
                lastReadDirectObject = "]";
                return FSTObjectOutput.DIRECT_OBJECT;
            }
            throw new RuntimeException("Expected Object start, got '"+jsonToken+"'");
        }

        String typeTag = input.nextFieldName();
        if ( typeTag.equals(fieldNames.TYPE) ) {
            // object
            String type = input.nextTextValue();
            String valueTag = input.nextFieldName();
            if ( ! fieldNames.OBJ.equals(valueTag) ) {
                throw new RuntimeException("expected value attribute for object of type:"+type);
            }
            if ( ! input.nextToken().isStructStart() ) {
                throw new RuntimeException("expected struct start");
            }
            try {
                lastDirectClass = classForName(conf.getClassForCPName(type));
            } catch (ClassNotFoundException e) {
                FSTUtil.<RuntimeException>rethrow(e);
            }
            return FSTObjectOutput.OBJECT;
        } else if ( typeTag.equals(fieldNames.SEQ_TYPE) ) {
            // sequence
            String type = input.nextTextValue();
            try {
                lastDirectClass = classForName(conf.getClassForCPName(type));
                String valueTag = input.nextFieldName();
                if ( ! fieldNames.SEQ.equals(valueTag) ) {
                    throw new RuntimeException("expected value attribute for object of type:"+type);
                }
                if ( ! input.nextToken().isStructStart() ) {
                    throw new RuntimeException("expected array start");
                }
            } catch (ClassNotFoundException e) {
                FSTUtil.<RuntimeException>rethrow(e);
            }
            return FSTObjectOutput.ARRAY;
        } else if ( typeTag.equals(fieldNames.REF) ) {
            return FSTObjectOutput.HANDLE;
        } else if ( typeTag.equals(fieldNames.ENUM) ) {
            try {
                String clName = input.nextTextValue();
                Class aClass = classForName(conf.getClassForCPName(clName));
                String valueTag = input.nextFieldName();
                if ( ! fieldNames.VAL.equals(valueTag) ) {
                    throw new RuntimeException("expected value attribute for enum of type:"+clName);
                }
                String enumString = input.nextTextValue();
                lastReadDirectObject = Enum.valueOf(aClass,enumString);
                input.nextToken(); // object end
            } catch (ClassNotFoundException e) {
                FSTUtil.<RuntimeException>rethrow(e);
            }
            return FSTObjectOutput.DIRECT_OBJECT;
        }
        // fall back to unknown
        lastDirectClass = Unknown.class;
        unknownFallbackReadFieldName = typeTag;
        return FSTObjectOutput.OBJECT;
    }

    List tmpList;
    private List getTmpList() {
        if ( tmpList == null ) {
            tmpList = new ArrayList(32);
        } else {
            tmpList.clear();
        }
        return tmpList;
    }

    protected Unknown createUnknownArray() throws IOException {
        unknownNestLevel++;
        List arrayTokens = new ArrayList(14);
        JsonToken elem = input.nextToken();
        while ( ! elem.isStructEnd() ) {
            if ( elem == JsonToken.VALUE_NUMBER_INT ) {
                arrayTokens.add(input.getLongValue());
            } else if ( elem == JsonToken.VALUE_NUMBER_FLOAT ) {
                arrayTokens.add(input.getDoubleValue());
            } else if ( elem == JsonToken.VALUE_TRUE ) {
                arrayTokens.add(true);
            } else if ( elem == JsonToken.VALUE_FALSE ) {
                arrayTokens.add(false);
            } else if ( elem == JsonToken.VALUE_NULL ) {
                arrayTokens.add(null);
            } else if ( elem == JsonToken.VALUE_STRING ){
                arrayTokens.add(input.getText());
            } else if ( elem == JsonToken.START_OBJECT ) {
                arrayTokens.add( readUnknownObject() );
            } else if ( elem == JsonToken.START_ARRAY ) {
                arrayTokens.add( createUnknownArray() );
            } else {
                throw new RuntimeException("unexpected array content in Unknown array");
            }
            elem = input.nextValue();
        }
        unknownNestLevel--;
        return new Unknown().items(arrayTokens);
    }

    private Unknown readUnknownObject() throws IOException {
        Unknown unk = new Unknown();
        JsonToken elem = input.nextToken();
        boolean expectField = true;
        String field = null;
        while ( ! elem.isStructEnd() ) {
            if ( expectField ) {
                field = input.getValueAsString();
                expectField = false;
            } else {
                if ( elem == JsonToken.VALUE_NUMBER_INT ) {
                    unk.set(field,input.getLongValue());
                } else if ( elem == JsonToken.VALUE_NUMBER_FLOAT ) {
                    unk.set(field,input.getDoubleValue());
                } else if ( elem == JsonToken.VALUE_TRUE ) {
                    unk.set(field,true);
                } else if ( elem == JsonToken.VALUE_FALSE ) {
                    unk.set(field,false);
                } else if ( elem == JsonToken.VALUE_NULL ) {
                    unk.set(field,null);
                } else if ( elem == JsonToken.VALUE_STRING ){
                    unk.set(field,input.getText());
                } else if ( elem == JsonToken.START_OBJECT ) {
                    unk.set(field,readUnknownObject());
                } else if ( elem == JsonToken.START_ARRAY ) {
                    unk.set(field, createUnknownArray());
                } else {
                    throw new RuntimeException("unexpected array content in Unknown array");
                }
                expectField = true;
            }
            elem = input.nextToken();
        }
        return unk;
    }

    private Object createPrimitiveArrayFrom( List directObject ) {
        if ( directObject.size() == 0 || directObject.get(0) instanceof String == false ) {
            directObject.add(0,"int"); //fixme:slow
        }
        Class arrT = null;
        switch ((String)directObject.get(0)) {
            case "boolean": arrT = boolean.class; break;
            case "byte": arrT = byte.class; break;
            case "char": arrT = char.class; break;
            case "short": arrT = short.class; break;
            case "int": arrT = int.class; break;
            case "long": arrT = long.class; break;
            case "float": arrT = float.class; break;
            case "double": arrT = double.class; break;
            default:
                directObject.add(0,"dummy");
                arrT = String.class;
        }
        Object newObj = Array.newInstance(arrT, directObject.size()-1);
        for (int i = 0; i < directObject.size()-1; i++) {
            Object n = directObject.get(i+1);
            if (arrT == boolean.class )
                Array.setBoolean(newObj, i, (Boolean) n);
            else if (arrT == byte.class )
                Array.setByte(newObj, i, ((Number) n).byteValue());
            else if (arrT == char.class )
                Array.setChar(newObj, i, (char) ((Number)n).intValue());
            else if (arrT == short.class )
                Array.setShort(newObj, i, ((Number)n).shortValue());
            else if (arrT == int.class )
                Array.setInt(newObj, i, ((Number)n).intValue());
            else if (arrT == long.class )
                Array.setLong(newObj, i, ((Number)n).longValue());
            else if (arrT == float.class )
                Array.setFloat(newObj, i, ((Number)n).floatValue());
            else if (arrT == double.class )
                Array.setDouble(newObj, i, ((Number)n).doubleValue());
            else if (arrT == String.class ) {
                Array.set(newObj,i,n);
            } else {
                System.err.println("unexpected primitive array type:"+arrT);
            }
        }
        return newObj;
    }

    public List readJSonArr2List(List arrayTokens) throws IOException {
        JsonToken elem = input.nextToken();
        while ( ! elem.isStructEnd() ) {
            if ( elem == JsonToken.VALUE_NUMBER_INT ) {
                arrayTokens.add(input.getLongValue());
            } else if ( elem == JsonToken.VALUE_NUMBER_FLOAT ) {
                arrayTokens.add(input.getDoubleValue());
            } else if ( elem == JsonToken.VALUE_TRUE ) {
                arrayTokens.add(true);
            } else if ( elem == JsonToken.VALUE_FALSE ) {
                arrayTokens.add(false);
            } else if ( elem == JsonToken.VALUE_NULL ) {
                arrayTokens.add(null);
            } else {
                arrayTokens.add(input.getText());
            }
            elem = input.nextValue();
        }
        return arrayTokens;
    }

    public Object getDirectObject() // in case class already resolves to read object (e.g. mix input)
    {
        Object tmp = lastReadDirectObject;
        lastReadDirectObject = null;
        return tmp;
    }

    Object lastReadDirectObject; // in case readClass already reads full minbin value
    @Override
    public FSTClazzInfo readClass() throws IOException, ClassNotFoundException {
        if (lastDirectClass != null ) {
            FSTClazzInfo clInfo = conf.getCLInfoRegistry().getCLInfo(lastDirectClass, conf);
            lastDirectClass = null;
            return clInfo;
        }
        return null;
    }

    @Override
    public Class classForName(String name) throws ClassNotFoundException {
        Class aClass = clzCache.get(name);
        if ( aClass == Unknown.class )
            lastUnknown = name;
        if (aClass!=null)
            return aClass;
        aClass = conf.getClassRegistry().classForName(name,conf);
        if ( aClass == Unknown.class )
            lastUnknown = name;
        clzCache.put(name,aClass);
        return aClass;
    }

    @Override
    public void registerClass(Class possible) {
//        throw new RuntimeException("not implemented");
    }

    @Override
    public void close() {
        //nothing to do (?)
//        throw new RuntimeException("not implemented");
    }

    @Override
    public void skip(int n) {
        throw new RuntimeException("not implemented");
    }

    @Override
    public void readPlainBytes(byte[] b, int off, int len) {
        try {
            for (int i = 0; i < len; i++) {
                input.nextToken();
                b[i+off] = input.getByteValue();
            }
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    @Override
    public boolean isMapBased() {
        return true;
    }

    public void consumeEndMarker() { // empty as flawed in minbin impl
    }

    @Override
    public Object readArrayHeader() throws Exception {
        if ( lastDirectClass == null ) {
            JsonToken jsonToken = input.nextToken();
            String type = null;
            if ( jsonToken == JsonToken.START_ARRAY ) {
                if (unknownNestLevel>0) {
                    return createUnknownArray();
                } else {
                    // direct primitive array [1,2, ..]
                    return createPrimitiveArrayFrom(readJSonArr2List(getTmpList()));
                }
            } else if ( jsonToken == JsonToken.VALUE_NULL ) {
                return null;
            } else {
                jsonToken = input.nextToken(); // seqType
                if ( fieldNames.TYPE.equals(input.getText())) {
                    // object
                    type = input.nextTextValue();
                    String valueTag = input.nextFieldName();
                    if ( !fieldNames.OBJ.equals(valueTag) ) {
                        throw new RuntimeException("expected value attribute for object of type:"+type);
                    }
                    return classForName(conf.getClassForCPName(type));
                }
                if (!fieldNames.SEQ_TYPE.equals(input.getText()) ) {
                    System.out.println(">" + input.getCurrentToken()+" "+input.getText());
                    input.nextToken();
                    System.out.println(">" + input.getCurrentToken()+" "+input.getText());
                    input.nextToken();
                    System.out.println(">" + input.getCurrentToken()+" "+input.getText());
                    input.nextToken();
                    System.out.println(">" + input.getCurrentToken()+" "+input.getText());
                    throw new RuntimeException("expected seqType");
                }
                jsonToken = input.nextToken(); // seqType : ""
                type = input.getText();
                jsonToken = input.nextToken(); // seq
                jsonToken = input.nextToken(); // seq : [
            }
            return classForName(conf.getClassForCPName(type));
        }
        Class ldc = this.lastDirectClass;
        if ( ldc == Unknown.class ) {
            ldc = Object[].class;
        }
        this.lastDirectClass = null; // marker, only valid once
        return ldc;
    }

    @Override
    public void readExternalEnd() {
        consumeEndMarker();
    }

    @Override
    public boolean isEndMarker(String s) {
        return s == null || s.equals("}") || s.equals("]");
    }

    @Override
    public int readVersionTag() throws IOException {
        return 0; // versioning not supported for json
    }

    @Override
    public void pushBack(int bytes) {
        //fstInput.psetPos(input.getPos()-bytes);
        throw new RuntimeException("not supported");
    }

    private void consumeEnd() {
        try {
            JsonToken jsonToken = input.nextToken();
            if ( jsonToken == null )
                return;
            if ( ! jsonToken.isStructEnd() ) {
                throw new RuntimeException("end of structure expected found:"+jsonToken+" : value:"+input.getValueAsString()+" fname:'"+input.getCurrentName()+"'");
            }
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    @Override
    public void readArrayEnd(FSTClazzInfo clzSerInfo) {
        try {
            JsonToken jsonToken = input.nextToken(); // ]
            if ( jsonToken == JsonToken.END_ARRAY ) { //&& (clzSerInfo == null || clzSerInfo.getSer() == null) ) { // need to read only 1 in case of custom ser
                jsonToken = input.nextToken();    // }
            } else {
                //System.out.println("debug "+clzSerInfo);
            }
            Object dummyDebug = jsonToken;
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
    }

    @Override
    public void readObjectEnd() {
        consumeEnd();
    }

    @Override
    public Object coerceElement(Class arrType, Object value) {
        if ( arrType == byte[].class && value instanceof String ) {
            try {
                return ((String) value).getBytes("UTF-8");
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
        }
        if ( value instanceof Number ) {
            Number n = (Number) value;
            if ( arrType == Byte.class ) {
                return new Byte(n.byteValue());
            } else if ( arrType == Short.class ) {
                return new Short(n.shortValue());
            } else if ( arrType == Integer.class ) {
                return new Integer(n.intValue());
            } else if ( arrType == Long.class ) {
                return new Long(n.longValue());
            } else if ( arrType == Double.class ) {
                return new Double(n.doubleValue());
            } else if ( arrType == Float.class ) {
                return new Float(n.floatValue());
            } else if ( arrType == Character.class ) {
                return new Character((char) n.intValue());
            }
        }
        return value;
    }

    @Override
    public int available() {
        fstInput.ensureReadAhead(1);
        return fstInput.available();
    }

    @Override
    public boolean inArray() {
        return input.getParsingContext().inArray();
    }

    @Override
    public void startFieldReading(Object newObj) {
        if ( newObj instanceof Unknown ) {
            ((Unknown) newObj).setType(lastUnknown);
            lastUnknown = null;
            unknownNestLevel++;
        }
    }

    @Override
    public void endFieldReading(Object newObj) {
        if ( newObj instanceof Unknown ) {
            unknownNestLevel--;
        }
    }

}


/*
 * Copyright 2014 Ruediger Moeller.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.nustaq.serialization.simpleapi;

import org.nustaq.serialization.*;
import org.nustaq.serialization.util.FSTUtil;

import java.io.IOException;

/**
 * Created by ruedi on 09.11.14.
 *
 * Encodes Objects to byte arrays and vice versa using slight value compression and a platform neutral data
 * layout (no diff regarding big/little endian). Implementation is conservative (no unsafe)
 *
 * As this makes use of the stream oriented API, operation is not zero copy. However this is not too significant
 * compared to cost of serialization.
 *
 * KEEP and reuse instances, creation is expensive.
 *
 * This class cannot be used concurrently.
 *
 * Works similar to the unsafe coders, but does not use Unsafe. Note that reading and writing
 * coder must match each other in type and configuration.
 *
 */
public class DefaultCoder implements FSTCoder {

    protected FSTConfiguration conf;
    FSTObjectInput input;
    FSTObjectOutput output;

    public DefaultCoder(boolean shared, Class ... toPreRegister) {
        conf = FSTConfiguration.createDefaultConfiguration();
        conf.setShareReferences(shared);
        if ( toPreRegister != null && toPreRegister.length > 0 ) {
            conf.registerClass(toPreRegister);
        }
        if ( shared ) {
            input = new FSTObjectInput(conf);
            output = new FSTObjectOutput(conf);
        } else {
            input = new FSTObjectInputNoShared(conf);
            output = new FSTObjectOutputNoShared(conf);
        }
    }

    public DefaultCoder( Class ... preregister ) {
        this(true, preregister);
    }

    public DefaultCoder() {
        this(true);
    }

    /**
     * will throw an FSTBufferTooSmallException if buffer is too small.
     */
    public int toByteArray( Object obj, byte result[], int resultOffset, int avaiableSize ) {
        output.resetForReUse();
        try {
            output.writeObject(obj);
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
        int written = output.getWritten();
        if ( written > avaiableSize ) {
            throw FSTBufferTooSmallException.Instance;
        }
        System.arraycopy(output.getBuffer(),0,result,resultOffset, written);
        return written;
    }

    public byte[] toByteArray( Object o ) {
        output.resetForReUse();
        try {
            output.writeObject(o);
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
        return output.getCopyOfWrittenBuffer();
    }

    @Override
    public FSTConfiguration getConf() {
        return conf;
    }

    public Object toObject( byte arr[], int off, int len ) {
        try {
            if ( off == 0 ) {
                input.resetForReuseUseArray(arr);
            } else {
                input.resetForReuseCopyArray(arr, off, len);
            }
            return input.readObject();
        } catch (Exception e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
        return null;
    }

    public Object toObject( byte arr[] ) {
        return toObject(arr,0,arr.length);
    }

}


/*
 * Copyright 2014 Ruediger Moeller.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.nustaq.serialization.coders;

import java.io.*;
import org.nustaq.offheap.bytez.*;
import org.nustaq.offheap.bytez.onheap.*;
import org.nustaq.serialization.*;
import org.nustaq.serialization.util.*;

/**
 * Created by ruedi on 09.11.2014.
 *
 * uses unsafe to read values directly from memory instead an inputstream. no value compression is applied.
 *
 */
public class FSTBytezDecoder  implements FSTDecoder {

    BasicBytez input;
    HeapBytez ascStringCache;
    FSTConfiguration conf;
    public FSTClazzNameRegistry clnames;
    long pos;
    InputStream inputStream;
    long readUntil = 0;

    public FSTBytezDecoder(FSTConfiguration conf, BasicBytez input) {
        this(conf);
        if ( FSTConfiguration.isAndroid )
            throw new RuntimeException("not supported on android");
        this.input = input;
    }

    @Override
    public void setConf(FSTConfiguration conf) {
        this.conf = conf;
    }

    public FSTBytezDecoder(FSTConfiguration conf) {
        this.conf = conf;
        clnames = (FSTClazzNameRegistry) conf.getCachedObject(FSTClazzNameRegistry.class);
        if (clnames == null) {
            clnames = new FSTClazzNameRegistry(conf.getClassRegistry());
        } else {
            clnames.clear();
        }
    }

    byte tmp[];
    public int ensureReadAhead(int bytes) {
        if ( inputStream != null ) {
            if ( pos+bytes > readUntil ) {
                return readNextInputChunk(bytes);
            }
        } else if ( pos+bytes > input.length() ) {
            return -1;
        }
        return 0;
    }

    protected int readNextInputChunk(int bytes) {
        try {
            int toRead = Math.min(Integer.MAX_VALUE - 5, bytes);
            if ( inputStream instanceof ByteArrayInputStream ) {
                toRead = Math.min(((ByteArrayInputStream) inputStream).available(),toRead);
            }
            if ( tmp == null || tmp.length < toRead ) {
                tmp = new byte[toRead];
            }
            int read = inputStream.read(tmp, 0, toRead);
            if ( read > 0 ) {
                if ( input.length() < pos+read ) {
                    BasicBytez bytez = input.newInstance(2*(pos + read));
                    input.copyTo(bytez,0,0,pos);
                    input = (HeapBytez) bytez;
                }
                input.set(pos,tmp,0,read);
                readUntil = pos+read;
                return read;
            } else if ( read == -1 )
                return -1;
            // fixme: should loop in case read == 0
        } catch (IOException e) {
            FSTUtil.<RuntimeException>rethrow(e);
        }
        return 0;
    }

    char chBufS[];
    char[] getCharBuf(int siz) {
        char chars[] = chBufS;
        if (chars == null || chars.length < siz) {
            chars = new char[Math.max(siz, 15)];
            chBufS = chars;
        }
        return chars;
    }

    public String readStringUTF() throws IOException {
        int len = readFInt();
        char[] charBuf = getCharBuf(len * 2);
        ensureReadAhead(len*2);
        input.getCharArr(pos,charBuf,0,len);
        pos += len*2;
        return new String(charBuf, 0, len);
    }

    public byte readObjectHeaderTag() throws IOException {
        return readFByte();
    }

    /**
     * len < 127 !!!!!
     *
     * @return
     * @throws java.io.IOException
     */
    @Override
    public String readStringAsc() throws IOException {
        int len = readFInt();
        if (ascStringCache == null || ascStringCache.length() < len)
            ascStringCache = new HeapBytez(new byte[len]);
        ensureReadAhead(len);
//        System.arraycopy(input.buf, input.pos, ascStringCache, 0, len);
        input.copyTo(ascStringCache, 0, pos, len);
        pos += len;
        return new String(ascStringCache.getBase(), 0, 0, len);
    }

    public BasicBytez getInput() {
        return input;
    }

    public void setInput(BasicBytez input) {
        this.input = input;
    }

    /**
     * assumes class header+len already read
     *
     * @param componentType
     * @param len
     * @return
     */
    @Override
    public Object readFPrimitiveArray(Object array, Class componentType, int len) {
        // FIXME: if else chaining could be avoided
        if (componentType == byte.class) {
            ensureReadAhead(len);
            byte arr[] = (byte[]) array;
            input.getArr(pos, arr, 0, len);
            pos += len;
            return arr;
        } else if (componentType == char.class) {
            ensureReadAhead(len*2);
            char[] arr = (char[]) array;
            input.getCharArr(pos, arr, 0, len);
            pos += len * 2;
            return arr;
        } else if (componentType == short.class) {
            ensureReadAhead(len*2);
            short[] arr = (short[]) array;
            input.getShortArr(pos, arr, 0, len);
            pos += len * 2;
            return arr;
        } else if (componentType == int.class) {
            ensureReadAhead(len*4);
            int[] arr = (int[]) array;
            input.getIntArr(pos, arr, 0, len);
            pos += len * 4;
            return arr;
        } else if (componentType == float.class) {
            ensureReadAhead(len*4);
            float[] arr = (float[]) array;
            input.getFloatArr(pos, arr, 0, len);
            pos += len * 4;
            return arr;
        } else if (componentType == double.class) {
            ensureReadAhead(len*8);
            double[] arr = (double[]) array;
            input.getDoubleArr(pos, arr, 0, len);
            pos += len * 8;
            return arr;
        } else if (componentType == long.class) {
            ensureReadAhead(len*8);
            long[] arr = (long[]) array;
            input.getLongArr(pos, arr, 0, len);
            pos += len * 8;
            return arr;
        } else if (componentType == boolean.class) {
            ensureReadAhead(len);
            boolean[] arr = (boolean[]) array;
            input.getBooleanArr(pos, arr, 0, len);
            pos += len;
            return arr;
        } else {
            throw new RuntimeException("unexpected primitive type " + componentType.getName());
        }
    }

    @Override
    public void readFIntArr(int len, int[] arr) throws IOException {
        input.getIntArr(pos, arr, 0, len);
        pos += len * 4;
    }

    @Override
    public int readFInt() throws IOException {
        return readPlainInt();
    }

    @Override
    public double readFDouble() throws IOException {
        return Double.longBitsToDouble(readPlainLong());
    }

    /**
     * Reads a 4 byte float.
     */
    @Override
    public float readFFloat() throws IOException {
        return Float.intBitsToFloat(readPlainInt());
    }

    @Override
    public final byte readFByte() throws IOException {
        ensureReadAhead(1);
        return input.get(pos++);
    }

    @Override
    public int readIntByte() throws IOException {
        final int res = ensureReadAhead(1);
        if ( res == -1 )
            return -1;
        return input.get(pos++) & 0xff;
    }

    @Override
    public long readFLong() throws IOException {
        return readPlainLong();
    }

    @Override
    public char readFChar() throws IOException {
        return readPlainChar();
    }

    @Override
    public short readFShort() throws IOException {
        return readPlainShort();
    }

    private char readPlainChar() throws IOException {
        ensureReadAhead(2);
        char res = input.getChar(pos);
        pos += 2;
        return res;
    }

    private short readPlainShort() throws IOException {
        ensureReadAhead(2);
        short res = input.getShort(pos);
        pos += 2;
        return res;
    }

    @Override
    public int readPlainInt() throws IOException {
        ensureReadAhead(4);
        int res = input.getInt(pos);
        pos += 4;
        return res;
    }

    private long readPlainLong() throws IOException {
        ensureReadAhead(8);
        long res = input.getLong(pos);
        pos += 8;
        return res;
    }

    @Override
    public byte[] getBuffer() {
        if ( input instanceof HeapBytez && ((HeapBytez) input).getOffsetIndex() == 0 ) {
            return ((HeapBytez) input).asByteArray();
        }
        byte res[] = new byte[(int) pos];
        input.getArr(0,res,0, (int) pos);
        return res;
    }

    @Override
    public int getInputPos() {
        return (int) pos;
    }

    @Override
    public void moveTo(int position) {
        pos = position;
    }

    @Override
    public void reset() {
        pos = 0;
        clnames.clear();
        inputStream = null;
    }

    @Override
    public void setInputStream(InputStream in) {
        if ( in == FSTObjectInput.emptyStream ) {
            inputStream = null;
            readUntil = 0;
            return;
        }
        this.inputStream = in;
        clnames.clear();
        pos = 0;
        if ( input == null )
            input = new HeapBytez(new byte[4096]);
        readUntil = 0;
    }

    @Override
    public void resetToCopyOf(byte[] bytes, int off, int len) {
        inputStream = null;
        if ( input == null ) {
            byte[] base = new byte[len];
            input = new HeapBytez(base,0,len);
        }
        if ( input.length() < len )
        {
            input = (HeapBytez) input.newInstance(len);
        }
        input.set(0, bytes, off, len);
        pos = 0;
        clnames.clear();
    }

    @Override
    public void resetWith(byte[] bytes, int len) {
        inputStream = null;
        if ( input == null ) {
            input = new HeapBytez(bytes,0,len);
            return;
        }
        // suboptimal method for non heap backing
        if ( input.getClass() == HeapBytez.class ) {
            ((HeapBytez)input).setBase(bytes,0,len);
        } else {
            BasicBytez newBytez = input.newInstance(len);
            newBytez.set(0,bytes,0,len);
        }
        pos = 0;
        clnames.clear();
    }

    @Override
    public FSTClazzInfo readClass() throws IOException, ClassNotFoundException {
        return clnames.decodeClass(this,conf);
    }

    @Override
    public Class classForName(String name) throws ClassNotFoundException {
        return clnames.classForName(name,conf);
    }

    @Override
    public void registerClass(Class possible) {
        clnames.registerClass(possible,conf);
    }

    @Override
    public void close() {
        conf.returnObject(clnames);
    }

    @Override
    public void skip(int n) {
        pos += n;
    }

    @Override
    public void readPlainBytes(byte[] b, int off, int len) {
        ensureReadAhead(len);
//        System.arraycopy(input.buf,input.pos,b,off,len);
        input.getArr(pos, b, off, len);
        pos += len;
    }

    @Override
    public boolean isMapBased() {
        return false;
    }

    @Override
    public Object getDirectObject() // in case class already resolves to read object (e.g. mix input)
    {
        return null;
    }

    @Override
    public int getObjectHeaderLen() {
        return -1;
    }

    @Override
    public void consumeEndMarker() {
    }

    @Override
    public Class readArrayHeader() throws Exception {
        return readClass().getClazz();
    }

    @Override
    public void readExternalEnd() {
        // do nothing for direct encoding
    }

    @Override
    public boolean isEndMarker(String s) {
        return false;
    }

    @Override
    public int readVersionTag() throws IOException {
        return readFByte();
    }

    @Override
    public void pushBack(int bytes) {
        pos -= bytes;
    }

    @Override
    public void readArrayEnd(FSTClazzInfo clzSerInfo) {}

    @Override
    public void readObjectEnd() {}

    @Override
    public Object coerceElement(Class arrType, Object value) {
        return value;
    }

    @Override
    public int available() {
        return (int) (input.length()-pos);
    }

    @Override
    public boolean inArray() {
        return false;
    }

    @Override
    public void startFieldReading(Object newObj) {

    }

    @Override
    public void endFieldReading(Object newObj) {

    }

}


/*
 * Copyright 2014 Ruediger Moeller.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.nustaq.serialization.util;

import sun.misc.Unsafe;

import java.io.InputStream;
import java.io.ObjectStreamField;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Arrays;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: ruedi
 * Date: 29.11.12
 * Time: 20:38
 * To change this template use File | Settings | File Templates.
 */
public class FSTUtil {

    static Object[] EmptyObjArray = new Object[10000];
    static ObjectStreamField[] NO_FIELDS = new ObjectStreamField[0];
    public static Unsafe unFlaggedUnsafe = FSTUtil.getUnsafe(); // even if unsafe is disabled, use it for memoffset computation

    static {
        if (unFlaggedUnsafe != null) {
            refoff = unFlaggedUnsafe.arrayBaseOffset(Object[].class);
            bufoff = unFlaggedUnsafe.arrayBaseOffset(byte[].class);
            intoff = unFlaggedUnsafe.arrayBaseOffset(int[].class);
            longoff = unFlaggedUnsafe.arrayBaseOffset(long[].class);
            longscal = unFlaggedUnsafe.arrayIndexScale(long[].class);
            intscal = unFlaggedUnsafe.arrayIndexScale(int[].class);
            chscal = unFlaggedUnsafe.arrayIndexScale(char[].class);
            refscal = unFlaggedUnsafe.arrayIndexScale(Object[].class);
            choff = unFlaggedUnsafe.arrayBaseOffset(char[].class);
            doubleoff = unFlaggedUnsafe.arrayBaseOffset(double[].class);
            doublescal = unFlaggedUnsafe.arrayIndexScale(double[].class);
            floatoff = unFlaggedUnsafe.arrayBaseOffset(float[].class);
            floatscal = unFlaggedUnsafe.arrayIndexScale(float[].class);
        } else {
            refscal = 0;
            refoff = 0;
            longoff = 0;
            longscal = 0;
            bufoff = 0;
            intoff = 0;
            intscal = 0;
            choff = 0;
            chscal = 0;
            doublescal = 0;
            doubleoff = 0;
            floatscal = 0;
            floatoff = 0;
        }
    }

    public final static long refoff;
    public final static long refscal;
    public final static long bufoff;
    public final static long choff;
    public final static long intoff;
    public final static long longoff;
    public final static long doubleoff;
    public final static long floatoff;
    public final static long intscal;
    public final static long longscal;
    public final static long chscal;
    public final static long floatscal;
    public final static long doublescal;

    public static void clear(int[] arr) {
        Arrays.fill(arr, 0);
    }

    public static void clear(Object[] arr) {
        final int arrlen = arr.length;
        clear(arr, arrlen);
    }

    public static void clear(Object[] arr, int arrlen) {
        int count = 0;
        final int length = EmptyObjArray.length;
        while (arrlen - count > length) {
            System.arraycopy(EmptyObjArray, 0, arr, count, length);
            count += length;
        }
        System.arraycopy(EmptyObjArray, 0, arr, count, arrlen - count);
    }

    public static String toString(Throwable th) {
        StringWriter sw = new StringWriter();
        PrintWriter pw = new PrintWriter(sw);
        th.printStackTrace(pw);
        return th.getClass().getSimpleName() + ":" + th.getMessage() + "\n" + sw.toString();
    }

    public static <T extends Throwable> void rethrow(Throwable exception) throws T {
        throw (T) exception;
    }

    // obsolete
    public static String getPackage(Class clazz) {
        String s = clazz.getName();
        int i = s.lastIndexOf('[');
        if (i >= 0) {
            s = s.substring(i + 2);
        }
        i = s.lastIndexOf('.');
        if (i >= 0) {
            return s.substring(0, i);
        }
        return "";
    }

    public static boolean isPackEq(Class clazz1, Class clazz2) {
        return getPackage(clazz1).equals(getPackage(clazz2));
    }

    public static Method findPrivateMethod(Class clazz, String methName,
                                           Class[] clazzArgs,
                                           Class retClazz) {
        try {
            Method m = clazz.getDeclaredMethod(methName, clazzArgs);
            int modif = m.getModifiers();
            if ((m.getReturnType() == retClazz) && ((modif & Modifier.PRIVATE) != 0) && ((modif & Modifier.STATIC) == 0)) {
                m.setAccessible(true);
                return m;
            }
            return null;
        } catch (NoSuchMethodException ex) {
            return null;
        }
    }

    public static Method findDerivedMethod(Class clazz, String metnam,
                                           Class[] argClzz,
                                           Class retClz) {
        Method m = null;
        Class defCl = clazz;
        while (defCl != null) {
            try {
                m = defCl.getDeclaredMethod(metnam, argClzz);
                break;
            } catch (NoSuchMethodException ex) {
                defCl = defCl.getSuperclass();
            }
        }
        if (m == null) {
            return null;
        }
        if (m.getReturnType() != retClz) {
            return null;
        }
        int mods = m.getModifiers();
        if ((mods & (Modifier.STATIC | Modifier.ABSTRACT)) != 0) {
            return null;
        } else if ((mods & (Modifier.PUBLIC | Modifier.PROTECTED)) != 0) {
            m.setAccessible(true);
            return m;
        } else if ((mods & Modifier.PRIVATE) != 0) {
            m.setAccessible(true);
            if (clazz == defCl) {
                return m;
            }
            return null;
        } else {
            m.setAccessible(true);
            if (isPackEq(clazz, defCl)) {
                return m;
            }
            return null;
        }
    }

    public static void printEx(Throwable e) {
        while (e.getCause() != null && e.getCause() != e) {
            e = e.getCause();
        }
        e.printStackTrace();
    }

    public static boolean isPrimitiveArray(Class c) {
        Class componentType = c.getComponentType();
        if (componentType == null) {
            return c.isPrimitive();
        }
        return isPrimitiveArray(c.getComponentType());
    }

    public static Unsafe getUnsafe() {
        try {
            if (unFlaggedUnsafe != null)
                return unFlaggedUnsafe;
            Field f = Unsafe.class.getDeclaredField("theUnsafe");
            f.setAccessible(true);
            return (Unsafe) f.get(null);
        } catch (Exception e) { /* ... */ }
        return null;
    }

    public static int writeSignedVarInt(int value, byte out[], int index) {
        return writeUnsignedVarInt((value << 1) ^ (value >> 31), out, index);
    }

    public static int writeUnsignedVarInt(int value, byte[] out, int index) {
        while ((value & 0xFFFFFF80) != 0L) {
            out[index++] = (byte) ((value & 0x7F) | 0x80);
            value >>>= 7;
        }
        out[index++] = (byte) (value & 0x7F);
        return index;
    }

    public static List<Field> getAllFields(List<Field> fields, Class<?> type) {
        fields.addAll(Arrays.asList(type.getDeclaredFields()));
        if (type.getSuperclass() != null) {
            fields = getAllFields(fields, type.getSuperclass());
        }
        return fields;
    }

    public static byte[] readAll(InputStream is)
            throws Exception {
        int pos = 0;
        byte[] buffer = new byte[1024];
        while (true) {
            int toRead;
            if (pos >= buffer.length) {
                toRead = buffer.length * 2;
                if (buffer.length < pos + toRead) {
                    buffer = Arrays.copyOf(buffer, pos + toRead);
                }
            } else {
                toRead = buffer.length - pos;
            }
            int byt = is.read(buffer, pos, toRead);
            if (byt < 0) {
                if (pos != buffer.length) {
                    buffer = Arrays.copyOf(buffer, pos);
                }
                break;
            }
            pos += byt;
        }
        return buffer;
    }

    public static int nextPow2(int num) {
        return 1 << (num == 0 ? 0 : 32 - Integer.numberOfLeadingZeros(num - 1));
    }

    public static Class getRealEnumClass(Class enumClass) {
        if (enumClass.isAnonymousClass()) {
            return enumClass.getSuperclass();
        }
        return enumClass;
    }
}


/*
 * Copyright 2014 Ruediger Moeller.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.nustaq.offheap.bytez.malloc;

import sun.misc.Cleaner;
import sun.nio.ch.FileChannelImpl;

import java.io.File;
import java.io.RandomAccessFile;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.channels.FileChannel;

/**
 * Bytez allocated inside a memory mapped file. Some Mmap file stuff handling is copied from OpenHFT library (too big to depend on for fst),
 * a great tool for all kind of binary/low level java stuff. Check it out at github.
 */
public class MMFBytez extends MallocBytez {
    private File file;
    private FileChannel fileChannel;
    private Cleaner cleaner;

    public MMFBytez(String filePath, long length, boolean clearFile) throws Exception {
        super(0, 0);
        init( filePath, length, clearFile );
    }

    protected void init(String file, long length, boolean clearFile) throws Exception {
        File f = new File(file);
        if ( f.exists() && clearFile ) {
            f.delete();
        }
        this.file = f;

        if ( f.exists() ) {
            length = f.length();
        }

        RandomAccessFile raf = new RandomAccessFile(f, "rw");
        raf.setLength(length); // FIXME: see stackoverflow. does not work always
        FileChannel fileChannel = raf.getChannel();

        this.fileChannel = raf.getChannel();
        this.baseAdress = map0(fileChannel, imodeFor(FileChannel.MapMode.READ_WRITE), 0L, length);
        this.length = length;
        this.cleaner = Cleaner.create(this, new Unmapper(baseAdress, length, fileChannel));
    }

    public void freeAndClose() {
        cleaner.clean();
    }

    /**
     * hack to update underlying file in slices handed out to app
     */
    public void _setMMFData(File file, FileChannel fileChannel,Cleaner cleaner) {
        this.file = file; this.fileChannel = fileChannel; this.cleaner = cleaner;
    }

    public File getFile() {
        return file;
    }

    public FileChannel getFileChannel() {
        return fileChannel;
    }

    public Cleaner getCleaner() {
        return cleaner;
    }

/**
     * stuff copied from OpenHFT library (too big to depend on for fst) ...
     *
     * Copyright 2013 Peter Lawrey
     *
     * Licensed under the Apache License, Version 2.0 (the "License");
     * you may not use this file except in compliance with the License.
     * You may obtain a copy of the License at
     *
     *         http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    /**
     * @param fileChannel
     * @param imode
     * @param start
     * @param size
     * @return
     * @throws NoSuchMethodException
     * @throws IllegalAccessException
     * @throws InvocationTargetException
     */

    private static final int MAP_RO = 0;
    private static final int MAP_RW = 1;
    private static final int MAP_PV = 2;

    private static long map0(FileChannel fileChannel, int imode, long start, long size) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        Method map0 = fileChannel.getClass().getDeclaredMethod("map0", int.class, long.class, long.class);
        map0.setAccessible(true);
        return (Long) map0.invoke(fileChannel, imode, start, size);
    }

    private static void unmap0(long address, long size) throws Exception {
        Method unmap0 = FileChannelImpl.class.getDeclaredMethod("unmap0", long.class, long.class);
        unmap0.setAccessible(true);
        unmap0.invoke(null, address, size);
    }

    private static int imodeFor(FileChannel.MapMode mode) {
        int imode = -1;
        if (mode == FileChannel.MapMode.READ_ONLY)
            imode = MAP_RO;
        else if (mode == FileChannel.MapMode.READ_WRITE)
            imode = MAP_RW;
        else if (mode == FileChannel.MapMode.PRIVATE)
            imode = MAP_PV;
        assert (imode >= 0);
        return imode;
    }

    static class Unmapper implements Runnable {
        private final long size;
        private final FileChannel channel;
        private volatile long address;

        Unmapper(long address, long size, FileChannel channel) {
            assert (address != 0);
            this.address = address;
            this.size = size;
            this.channel = channel;
        }

        public void run() {
            if (address == 0)
                return;

            try {
                unmap0(address, size);
                address = 0;

                if (channel.isOpen()) {
                    channel.close();
                }

            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}


package org.nustaq.serialization;

import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

/**
 * Represents a ordered class lineage based on the specificity of classes, where specificity is defined as follows:
 * <nl>
 *
null has specificity 0

 *
java.lang.Object has specificity 0

 *
an interface without any extends clause has specificity 1

 *
a class or interface has a specificity of 1 + the specificity of the superclass + the sum of the specificity of the implemented interfaces.

 * </nl>
 * @author Odd Moeller 2017-03-08.
 */
public final class FSTClazzLineageInfo {
  private static final Class<?>[] EMPTY_CLASS_ARRAY = new Class<?>[0];
  private static final LineageInfo OBJECT_LINEAGE_INFO = new LineageInfo(new LinkedHashSet<Class<?>>(Collections.singletonList(Object.class)), 0);
  private static final ConcurrentMap<Class<?>, LineageInfo> lineageInfos = new ConcurrentHashMap<Class<?>, LineageInfo>();
  public static final Comparator<Class> SPECIFICITY_CLASS_COMPARATOR = new Comparator<Class>() {
    @Override
    public int compare(final Class c1, final Class c2) {
      return getLineageInfo(c2).specificity - getLineageInfo(c1).specificity;
    }
  };

  private FSTClazzLineageInfo() {}

  /**
   * Returns the specificity of the specified class as defined above.
   */
  public static int getSpecificity(final Class<?> clazz) {
    if (clazz == null) return 0;
    final LineageInfo lineageInfo = FSTClazzLineageInfo.getLineageInfo(clazz);
    return lineageInfo == null ? 0 : lineageInfo.specificity;
  }

  /**
   * Returns the lineage of the specified class ordered by specificity (the class itself is at position 0 since it is most specific in its lineage).
   */
  public static Class<?>[] getLineage(final Class<?> clazz) {
    final LineageInfo lineageInfo = getLineageInfo(clazz);
    return lineageInfo == null ? EMPTY_CLASS_ARRAY : lineageInfo.lineage.toArray(new Class<?>[lineageInfo.lineage.size()]);
  }

  private static LineageInfo getLineageInfo(final Class<?> clazz) {
    if (clazz == null) return null;
    else if (clazz.equals(Object.class)) return OBJECT_LINEAGE_INFO;
    final LineageInfo lineageInfo = lineageInfos.get(clazz);
    if (lineageInfo != null) {
      return lineageInfo;
    }

    int specificity = 1;
    final LinkedHashSet<Class<?>> ancestors = new LinkedHashSet<Class<?>>();
    final Class<?> sc = getSuperclass(clazz);
    final LineageInfo sl = getLineageInfo(sc);
    if (sl != null) {
      ancestors.addAll(sl.lineage);
      specificity += sl.specificity;
    }
    for (final Class<?> i : getInterfaces(clazz)) {
      final LineageInfo il = getLineageInfo(i);
      if (il != null) {
        ancestors.removeAll(il.lineage);
        ancestors.addAll(il.lineage);
        specificity += il.specificity;
      }
    }
    final Class<?>[] array = ancestors.toArray(new Class<?>[ancestors.size()]);
    Arrays.sort(array, SPECIFICITY_CLASS_COMPARATOR);
    final LinkedHashSet<Class<?>> lineage = new LinkedHashSet<Class<?>>(array.length + 1);
    lineage.add(clazz);
    Collections.addAll(lineage, array);
    final LineageInfo result = new LineageInfo(lineage, specificity);
    lineageInfos.putIfAbsent(clazz, result);
    return result;
  }

  private static Class<?> getSuperclass(final Class<?> c) {
    if (c == null) return null;
    return c.getSuperclass();
  }

  private static Class<?>[] getInterfaces(final Class<?> c) {
    if (c == null) return null;
    return c.getInterfaces();
  }

  private static final class LineageInfo {
    private final LinkedHashSet<Class<?>> lineage;
    private final int specificity;
    private LineageInfo(final LinkedHashSet<Class<?>> lineage, final int specificity) {
      this.lineage = lineage;
      this.specificity = specificity;
    }
  }
}


/*
 * Copyright 2014 Ruediger Moeller.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.nustaq.offheap;

import org.nustaq.offheap.bytez.ByteSource;
import org.nustaq.offheap.bytez.bytesource.AsciiStringByteSource;
import org.nustaq.offheap.bytez.bytesource.BytezByteSource;
import org.nustaq.offheap.bytez.Bytez;
import org.nustaq.offheap.bytez.malloc.MMFBytez;
import org.nustaq.offheap.bytez.malloc.MallocBytez;
import org.nustaq.offheap.bytez.malloc.MallocBytezAllocator;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ConcurrentModificationException;
import java.util.Iterator;

/**
 * Baseclass of offheap maps.
 * FST OffHeap Maps enable to store key/value pairs in offheap memory. Additionally it provides
 * and iterator interface for all values. In order to also iterate keys, add each key to its value object,
 * as this OffHeap map does not support iteration of keys out of the box.
 *
 * See subclasses for directly applicable classes (E.g. FSTStringOffheapMap)
 *
 * The base is a generic bytesource to bytesource map. Note that key should be as short as possible (4-20 bytes) and
 * should have their most modified digit at the last character of their value.
 * e.g. [0,0,0,0,123,44] where '44' changes with each new key. Else on-heap memory consumption will grow.
 * Performance of lookup degrades with growing key size.
 *
 * In case entries are updated frequently with values of different size, avoid fragementation
 * by adding extra space to each entry. Override getEntryLengthForContentLength for this.
 */
public class FSTBinaryOffheapMap {

    // FIXME: Testcase for mem overflow

    public static final long MB = 1024 * 1024;
    public static final long GB = 1024 * MB;
    public static final int CUSTOM_FILEHEADER_LEN = 8000; // 8k for application
    public static final int CORE_HEADER_LEN = 8;
    public static final int FILE_HEADER_LEN = CORE_HEADER_LEN +CUSTOM_FILEHEADER_LEN; // 0 - numelems, 4 - magic num

    final static int HEADER_TAG = 0xe5e1; // can be used to recover corrupted data
    public static final int KEY_OFFSET_IN_HEADER = 16;

    private BytezByteSource tmpValueBytez;

    protected OffHeapByteTree index;
    protected Bytez memory;
    protected Bytez customHeader;
    protected MallocBytezAllocator alloc;
    protected int numElem;
    protected int keyLen;
    protected long bytezOffset;
    protected FreeList freeList;// FIXME: missing merge/split of different block sizes
    protected String mappedFile;
    protected int mutationCount;

    public FSTBinaryOffheapMap(String mappedFile, int keyLen, long sizeMemBytes, int numberOfElems) throws Exception {
        initFromFile(mappedFile, keyLen, sizeMemBytes, numberOfElems);
    }

    Thread debug;
    private void checkThread() {
//        if ( debug == null )
//            debug = Thread.currentThread();
//        else {
//            if ( debug != Thread.currentThread() )
//                throw new RuntimeException( "unexpected concurrency "+debug.getName()+" curr:" + Thread.currentThread().getName() );
//        }
    }

    public Bytez getCustomFileHeader() {
        return customHeader;
    }

    protected void initFromFile(String file, int keyLen, long sizeMemBytes, int numberOfElems) throws Exception {
        checkThread();
        numElem = 0;
        bytezOffset = FILE_HEADER_LEN;
        freeList = new FreeList(); // FIXME: missing merge/split of different block sizes
        this.mappedFile = file;
        resetMem(file, sizeMemBytes);
        this.keyLen = keyLen;
        if ( memory.getInt(4) != HEADER_TAG || memory.getInt(0) <= 0 ) {
            // newly created or empty file
            index = new OffHeapByteTree(keyLen,OffHeapByteTree.estimateMBytesForIndex(keyLen,numberOfElems));
            memory.putInt(4,HEADER_TAG);
        } else {
            // FIXME: be more resilent in case of corruption ..
            numElem = memory.getInt(0);
            index = new OffHeapByteTree(keyLen,OffHeapByteTree.estimateMBytesForIndex(keyLen,numElem*2));
            long off = FILE_HEADER_LEN;
            int elemCount = 0;
            BytezByteSource byteIter = new BytezByteSource(memory,0,0);
//            BytezByteSource byteVal = new BytezByteSource(memory,0,0);

            while (elemCount < numElem) {
                int len = getLenFromHeader(off);
//                int contentLen = getContentLenFromHeader(off);

                boolean removed = memory.get(off+4) != 0;

                if ( ! removed ) {
                    elemCount++;
                    byteIter.setOff(off + KEY_OFFSET_IN_HEADER); // 16 = offset of key in header
                    byteIter.setLen(keyLen);
                    index.put(byteIter, off);
                    bytezOffset = off+getHeaderLen()+len;
                } else {
                    addToFreeList(off);
                }

//                for ( int i=0; i < byteIter.length(); i++ ) {
//                    System.out.print((char) byteIter.get(i));
//                }
//                System.out.println();

                off+= getHeaderLen() + len;
            }
        }
    }

    private void resetMem(String file, long sizeMemBytes) throws Exception {
        checkThread();
        mutationCount++;
        memory = new MMFBytez(file,sizeMemBytes,false);
        customHeader = memory.slice(CORE_HEADER_LEN, CUSTOM_FILEHEADER_LEN);
        tmpValueBytez = new BytezByteSource(memory,0,0);
    }

    public FSTBinaryOffheapMap(int keyLen, long sizeMemBytes, int numberOfElems) {
        init(keyLen, sizeMemBytes, numberOfElems);
    }

    protected void init(int keyLen, long sizeMemBytes, int numberOfElems) {
        checkThread();
        numElem = 0;
        bytezOffset = FILE_HEADER_LEN;
        freeList = new FreeList(); // FIXME: missing merge/split of different block sizes
        alloc = new MallocBytezAllocator();
        memory = alloc.alloc(sizeMemBytes);
        customHeader = memory.slice(CORE_HEADER_LEN,CUSTOM_FILEHEADER_LEN);
        tmpValueBytez = new BytezByteSource(memory,0,0);
        this.keyLen = keyLen;
        index = new OffHeapByteTree(keyLen,OffHeapByteTree.estimateMBytesForIndex(keyLen,numberOfElems));
        memory.putInt(4, HEADER_TAG);
    }

    @Override
    protected void finalize() throws Throwable {
        free();
    }

    public void free() {
        checkThread();
        mutationCount++;
        if ( alloc != null ) {
            alloc.freeAll();
            alloc = null;
        }
        if ( memory instanceof MMFBytez ) {
            ((MMFBytez) memory).freeAndClose();
            memory = null;
        }
//        index.free();
        index = null;
    }

    public void putBinary( ByteSource key, ByteSource value ) {
        checkThread();
        if ( key.length() != keyLen )
            throw new RuntimeException("key must have length "+keyLen);
        mutationCount++;
        long put = index.get(key);
        if ( put != 0 ) {
            int lenFromHeader = getLenFromHeader(put);
            if (value.length() <= lenFromHeader) {
                // replace
                setEntry(put, lenFromHeader, value);
                index.put(key, put);
                return;
            }
            // set removed and fall through to add
            index.put(key, addEntry(key, value));
            removeEntry(put);
        } else {
            // add
            index.put(key, addEntry(key, value));
            incElems();
        }
    }

    protected void removeEntry(long offset) {
        checkThread();
        mutationCount++;
        addToFreeList(offset);
        memory.put(offset + 4, (byte) 1);
    }

    protected void addToFreeList(long offset) {
        freeList.addToFree(offset, getLenFromHeader(offset) + getHeaderLen());
    }

    protected void setEntry(long off, int entryLen, ByteSource value) {
        checkThread();
        mutationCount++;
        writeEntryHeader(off, entryLen, (int) value.length(), false);
        off += getHeaderLen();
        for ( int i = 0; i < value.length(); i++ ) {
            memory.put( off++, value.get(i) );
        }
    }

    protected long addEntry(ByteSource key, ByteSource value) {
        checkThread();
        mutationCount++;
        long valueLength = value.length();
        long newOffset = freeList.findFreeBlock( (int) valueLength + getHeaderLen() );
        if ( newOffset > 0) {
//            System.out.println("reuse len "+getLenFromHeader(newOffset)+" at "+newOffset+" entrylen "+(getLenFromHeader(newOffset)+getHeaderLen()));
            writeEntryHeader(newOffset,getLenFromHeader(newOffset),(int) valueLength,false);
            long l = newOffset;
            // put key
            for ( int ii = 0; ii < keyLen; ii++ ) {
                memory.put( 16+l+ii, key.get(ii) );
            }
            l += getHeaderLen();
            // put value
            for ( int ii = 0; ii < valueLength; ii++ ) {
                memory.put( l++, value.get(ii) );
            }
            return newOffset;
        }
        int entryLen = getEntryLengthForContentLength(value.length());
        // size to power of 2
        entryLen = freeList.computeLen(entryLen+getHeaderLen())-getHeaderLen();
        if ( memory.length() <= bytezOffset+entryLen+getHeaderLen()) {
            resizeStore(bytezOffset + entryLen + getHeaderLen());
//            return addEntry(key,value); // fixme loses one freelist entry
        }
        long res = bytezOffset;
        writeEntryHeader(bytezOffset, entryLen,(int)value.length(),false);
        // put key
        for ( int ii = 0; ii < keyLen; ii++ ) {
            memory.put( 16+bytezOffset+ii, key.get(ii) );
        }
        long off = bytezOffset+getHeaderLen();
        for ( int i = 0; i < value.length(); i++ ) {
            memory.put( off++, value.get(i) );
        }
        bytezOffset+=entryLen+getHeaderLen();
        return res;
    }

    private void resizeStore(long required) {
        resizeStore(required,GB);
    }

    /**
     * PRIVILEGED method. You gotta know what your doing here ..
     *
     * currently a very expensive operation .. frees everything, resize file and remap.
     * Remapping involves rebuild of index.
     * @param required
     */
    public void resizeStore(long required, long maxgrowbytes) {
        if ( mappedFile == null )
            throw new RuntimeException("store is full. Required: "+required);
        if ( required <= memory.length() )
            return;
        mutationCount++;
        System.out.println("resizing underlying "+mappedFile+" to "+required+" numElem:"+numElem);
        long tim = System.currentTimeMillis();
        ((MMFBytez) memory).freeAndClose();
        memory = null;
        try {
            File mf = new File(mappedFile);
            FileOutputStream f = new FileOutputStream(mf,true);
            long len = mf.length();
            required = required + Math.min(required,maxgrowbytes);
            byte[] toWrite = new byte[1000];
            long max = (required - len)/1000;
            for ( long i = 0; i < max+2; i++ ) {
                f.write(toWrite);
            }
            f.flush();
            f.close();
            resetMem(mappedFile, mf.length());
            System.out.println("resizing done in "+(System.currentTimeMillis()-tim)+" numElemAfter:"+numElem);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * get an entry. the returned ByteSource must be processed immediately as it will be reused
     * internally on next get
     * Warning: Concurrent Modification (e.g. add remove elements during iteration) is NOT SUPPORTED
     * and NOT CHECKED. Collect keys to change inside iteration and perform changes after iteration is
     * finished.
     * @param key
     * @return
     */
    public BytezByteSource getBinary( ByteSource key ) {
        checkThread();
        if ( key.length() != keyLen )
            throw new RuntimeException("key must have length "+keyLen);
        long aLong = index.get(key);
        if ( aLong == 0 ) {
            return null;
        }
        long off = aLong;
        int len = getContentLenFromHeader(off);
        off+= getHeaderLen();
        tmpValueBytez.setLen(len);
        tmpValueBytez.setOff(off);
        return tmpValueBytez;
    }

    /**
     * remove the key from the binary map
     * @param key
     */
    public void removeBinary( ByteSource key ) {
        checkThread();
        if ( key.length() != keyLen )
            throw new RuntimeException("key must have length "+keyLen);
        mutationCount++;
        long rem = index.get(key);
        if ( rem != 0 ) {
            index.remove(key);
            decElems();
            removeEntry(rem);
        }
    }

    protected void decElems() {
        numElem--;
        memory.putInt(0, numElem);
    }

    protected void incElems() {
        numElem++;
        memory.putInt(0, numElem);
    }

    /**
     * called upon add, allows to reserve extra space for later growth per entry
     * @param lengthOfEntry
     * @return
     */
    protected int getEntryLengthForContentLength(long lengthOfEntry) {
        return (int) lengthOfEntry;
    }

    protected void writeEntryHeader( long offset, int entryLen, int contentLen, boolean removed ) {
        checkThread();
        mutationCount++;
        memory.putInt( offset, entryLen );
        memory.put( offset + 4, (byte) (removed ? 1 : 0));
        memory.putInt( offset + 8, contentLen);
        memory.putInt( offset + 12, HEADER_TAG);
    }

    protected int getHeaderLen() {
        return 4+4+4+4+keyLen; // 0-3 len, 4 removed flag, 5-7 free, 8-11 content len, 12-15 magic num, 16... key
    }

    // overall content size (excl header)
    protected int getLenFromHeader(long off) {
        return memory.getInt(off);
    }

    protected int getContentLenFromHeader(long off) {
        return memory.getInt(off+8);
    }

    public Iterator<ByteSource> binaryValues() {
        checkThread();
        return new Iterator<ByteSource>() {
            long off = FILE_HEADER_LEN;
            int elemCount = 0;
            int mutSnap = mutationCount;

            BytezByteSource byteIter = new BytezByteSource(memory,0,0);

            @Override
            public boolean hasNext() {
                return elemCount < numElem;
            }

            @Override
            public ByteSource next() {
                checkThread();
                int contentLen = getContentLenFromHeader(off);
                int len = getLenFromHeader(off);
                boolean removed = memory.get(off+4) != 0;
                off+= getHeaderLen();
                while ( removed ) {
                    off += len;
                    len = getLenFromHeader(off);
                    contentLen = getContentLenFromHeader(off);
                    removed = memory.get(off+4) != 0;
                    off+= getHeaderLen();
                }
                elemCount++;
                byteIter.setOff(off);
                byteIter.setLen(contentLen);
                off+=len;
                if ( mutSnap != mutationCount )
                    throw new ConcurrentModificationException("in offheap map snap:"+mutSnap+" current:"+mutationCount);
                return byteIter;
            }

            @Override
            public void remove() {
                throw new RuntimeException("unimplemented");
            }
        };
    }

    public String printBinaryKey(ByteSource key) {
        StringBuilder res = new StringBuilder();
        for ( int i = 0; i < key.length(); i++ ) {
            byte b = key.get(i);
            if ( b > 31 ) {
                res.append((char) b);
            } else {
                res.append('_');
            }
        }
        return res.toString();
    }

    public KeyValIter binaryKeys() {
        checkThread();
        return new KeyValIter() {
            long off = FILE_HEADER_LEN;
            int elemCount = 0;
            int mutSnap = mutationCount;
            BytezByteSource byteIter = new BytezByteSource(memory,0,0);
            BytezByteSource byteVal = new BytezByteSource(memory,0,0);
            long valueAddress;

            @Override
            public boolean hasNext() {
                return elemCount < numElem;
            }

            @Override
            public ByteSource next() {
                checkThread();
                int len = getLenFromHeader(off);
                int contentLen = getContentLenFromHeader(off);
                boolean removed = memory.get(off+4) != 0;
                off+= getHeaderLen();
                while ( removed ) {
                    off += len;
                    len = getLenFromHeader(off);
                    contentLen = getContentLenFromHeader(off);
                    removed = memory.get(off+4) != 0;
                    off+= getHeaderLen();
                }
                elemCount++;
                valueAddress = off;
                byteVal.setOff(off);
                byteVal.setLen(contentLen);
                byteIter.setOff(off-getHeaderLen()+16);
                byteIter.setLen(keyLen);
                off+=len;
                if ( mutSnap != mutationCount )
                    throw new ConcurrentModificationException("in offheap map snap:"+mutSnap+" current:"+mutationCount);
                return byteIter;
            }

            @Override
            public void remove() {
                throw new RuntimeException("unimplemented");
            }

            @Override
            public ByteSource getValueBytes() {
                return byteVal;
            }

            @Override
            public long getValueAddress() {
                return valueAddress;
            }
        };
    }

    public long getFreeMem() {
        return memory.length()-bytezOffset;
    }

    public long getUsedMem() {
        return bytezOffset;
    }

    public int getCapacityMB() { return (int) (memory.length()/1024l/1024l); }

    public int getSize() {
        return numElem;
    }

    public void dumpIndexStats() {
        index.dumpStats();
    }

    public String getFileName() {
        return mappedFile;
    }

    public static interface KeyValIter extends Iterator<ByteSource> {
        public ByteSource getValueBytes();
        public long getValueAddress();
    }
}