/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.dap4lib.netcdf;

import com.sun.jna.Pointer;
import dap4.core.data.DataCursor;
import dap4.core.dmr.*;
import dap4.core.util.*;
import dap4.dap4lib.AbstractCursor;
import dap4.dap4lib.LibTypeFcns;
import ucar.nc2.jni.netcdf.Nc4prototypes;
import ucar.nc2.jni.netcdf.SizeT;

import java.lang.reflect.Array;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import static dap4.dap4lib.netcdf.Nc4DSP.Nc4Pointer;
import static dap4.dap4lib.netcdf.Nc4Notes.*;

public class Nc4Cursor extends AbstractCursor
{

    //////////////////////////////////////////////////

    static public boolean DEBUG = false;

    //////////////////////////////////////////////////
    // Instance variables

    protected Nc4Pointer memory = null;

    //////////////////////////////////////////////////
    // Constructor(s)

    public Nc4Cursor(Scheme scheme, Nc4DSP dsp, DapVariable template, Nc4Cursor container)
            throws DapException
    {
        super(scheme, dsp, template, container);
        if(DEBUG) debug();
    }

    public Nc4Cursor(Nc4Cursor c)
    {
        super(c);
        assert false;
        this.memory = c.getMemory();
    }

    //////////////////////////////////////////////////
    // AbstractCursor Interface API Implementations

    @Override
    public Object
    read(Index index)
            throws DapException
    {
        return read(DapUtil.indexToSlices(index));
    }

    @Override
    public Object
    read(List<Slice> slices)
            throws DapException
    {
        switch (this.scheme) {
        case ATOMIC:
            return readAtomic(slices);
        case STRUCTURE:
        case SEQUENCE:
            if(((DapVariable) this.getTemplate()).getRank() > 0
                    || DapUtil.isScalarSlices(slices))
                throw new DapException("Cannot slice a scalar variable");
            return this;
        case STRUCTARRAY:
            Odometer odom = Odometer.factory(slices);
            DataCursor[] instances = new DataCursor[(int) odom.totalSize()];
            for(int i = 0; odom.hasNext(); i++) {
                instances[i] = readStructure(odom.next());
            }
            return instances;
        case SEQARRAY:
            odom = Odometer.factory(slices);
            instances = new DataCursor[(int) odom.totalSize()];
            for(int i = 0; odom.hasNext(); i++) {
                instances[i] = readSequence(odom.next());
            }
            return instances;
        default:
            throw new DapException("Attempt to slice a scalar object");
        }
    }

    @Override
    public Nc4Cursor
    readField(int findex)
            throws DapException
    {
        // Preliminaries
        DapVariable template = (DapVariable) getTemplate();
        DapStructure struct = (DapStructure) template.getBaseType();
        if(findex < 0 || findex >= struct.getFields().size())
            throw new DapException("Field index out of range: " + findex);
        DapVariable field = struct.getField(findex);
        // Get VarNotes and TypeNotes
        VarNotes fi = (VarNotes) ((Nc4DSP) getDSP()).find(field);
        TypeNotes ti = fi.getBaseType();
        Nc4Cursor cursor = new Nc4Cursor(schemeFor(field), (Nc4DSP) this.dsp, field, this);
        Nc4Pointer mem = getMemory();
        // Handle records and structures somewhat differently
        if(this.scheme == scheme.STRUCTURE) {
            // do nothing
        } else if(this.getScheme() == Scheme.RECORD) {
            if(findex != 0) // need to use the first field of the sequence
                throw new DapException("Field index out of range: " + findex);
            // For fields, the record memory and the field memory are the same
            // because the record contains only 1 rank-0 field.
        } else  //Error
            throw new DapException("readfield expected STRUCTURE or RECORD cursor");
        cursor.setMemory(mem);
        return cursor;
    }

    @Override
    public long
    getRecordCount()
    {
        assert (this.scheme == scheme.SEQUENCE);
        if(this.recordcount < 0)
            throw new IllegalStateException("Sequence has no record count");
        return this.recordcount;
    }

    @Override
    public Nc4Cursor
    readRecord(long recno)
            throws DapException
    {
        assert (this.scheme == scheme.SEQUENCE);
        DapVariable template = (DapVariable) getTemplate();
        if(recno < 0 || recno >= getRecordCount())
            throw new ArrayIndexOutOfBoundsException("Illegal record id: " + recno);
        VarNotes vn = (VarNotes) ((Nc4DSP) getDSP()).find(template);
        TypeNotes ti = vn.getBaseType();
        assert (ti.isVlen());
        DapStructure ds = (DapStructure) template.getBaseType();
        DapVariable field = ds.getField(0);
        DapType ftype = field.getBaseType();
        TypeNotes fnotes = (TypeNotes) ((Nc4DSP) getDSP()).find(ftype);
        // The memory is a vector of objects, where the object type
        // is the type of the first field of the sequence
        // We need to extract the recno'th object from our memory
        Nc4Pointer record = getMemory(); // should be the vector of records
        long recsize = getElementSize(fnotes);
        // point to the recno element in the vector
        record = record.share(recno * recsize, recsize);
        Nc4Cursor rec = new Nc4Cursor(Scheme.RECORD, (Nc4DSP) getDSP(), (DapVariable) getTemplate(), this);
/*
        Nc4Pointer seqvlen = getMemory();
        long vlensize = Nc4prototypes.Vlen_t.VLENSIZE;
        Nc4Pointer recnovlen = seqvlen.share(vlensize*recno,vlensize); // recno'th vlen
        Nc4prototypes.Vlen_t vleninstance = new Nc4prototypes.Vlen_t(recnovlen.p);
        vleninstance.read(); // get the actual vlen contents for this sequence
        Pointer vlenmem = vleninstance.p;
        int vlencount = vleninstance.len;
        TypeNotes fvtype = getVlenType(field);
        long elemsize = fvtype.getSize();  // vlen.p is vector of objects of this size
        Nc4Pointer objectvec = new Nc4Pointer(vlenmem, elemsize * vlencount);
        // Now, we want the recno'th object in the vector
        Nc4Pointer record = objectvec.share(elemsize * recno, elemsize);
*/
        rec.setMemory(record).setRecordIndex(recno);
        return rec;
    }

    @Override
    public Index
    getIndex()
            throws DapException
    {
        if(this.scheme != Scheme.STRUCTURE && this.scheme != Scheme.SEQUENCE)
            throw new DapException("Not a Sequence|Structure instance");
        return this.arrayindex;
    }

    //////////////////////////////////////////////////
    // Support Methods

    protected Object
    readAtomic(List<Slice> slices)
            throws DapException
    {
        if(slices == null)
            throw new DapException("DataCursor.read: null set of slices");
        assert (this.scheme == scheme.ATOMIC);
        DapVariable atomvar = (DapVariable) getTemplate();
        int rank = atomvar.getRank();
        assert slices != null && ((rank == 0 && slices.size() == 1) || (slices.size() == rank));
        // Get VarNotes and TypeNotes
        Notes n = ((Nc4DSP) this.dsp).find(this.template);
        Object result = null;
        long count = DapUtil.sliceProduct(slices);
        VarNotes vn = (VarNotes) n;
        TypeNotes ti = vn.getBaseType();
        if(getContainer() == null) {
            if(rank == 0) { //scalar
                result = readAtomicScalar(vn, ti);
            } else {
                result = readAtomicVector(vn, ti, count, slices);
            }
        } else {// field of a structure instance or record
            long elemsize = ((DapType) ti.get()).getSize();
            assert (this.container != null);
            long trueoffset = computeTrueOffset(this);
            Nc4Pointer varmem = getMemory();
            Nc4Pointer mem = varmem.share(trueoffset, count * elemsize);
            result = getatomicdata(ti.getType(), count, elemsize, mem);
        }
        return result;
    }

    /**
     * Read a top-level scalar atomic variable
     *
     * @param vi
     * @param ti
     * @return
     * @throws DapException
     */
    protected Object
    readAtomicScalar(VarNotes vi, TypeNotes ti)
            throws DapException
    {
        DapVariable atomvar = (DapVariable) getTemplate();
        // Get into memory
        Nc4prototypes nc4 = ((Nc4DSP) this.dsp).getJNI();
        int ret;
        DapType basetype = ti.getType();
        Object result = null;
        if(basetype.isFixedSize()) {
            long memsize = ((DapType) ti.get()).getSize();
            Nc4Pointer mem = Nc4Pointer.allocate(memsize);
            readcheck(nc4, ret = nc4.nc_get_var(vi.gid, vi.id, mem.p));
            setMemory(mem);
            result = getatomicdata(ti.getType(), 1, mem.size, mem);
        } else if(basetype.isStringType()) {
            String[] s = new String[1];
            readcheck(nc4, ret = nc4.nc_get_var_string(vi.gid, vi.id, s));
            result = s;
        } else if(basetype.isOpaqueType()) {
            Nc4Pointer mem = Nc4Pointer.allocate(ti.getSize());
            readcheck(nc4, ret = nc4.nc_get_var(vi.gid, vi.id, mem.p));
            setMemory(mem);
            ByteBuffer[] buf = new ByteBuffer[1];
            buf[0] = mem.p.getByteBuffer(0, ti.getSize());
            result = buf;
        } else
            throw new DapException("Unexpected atomic type: " + basetype);
        return result;
    }

    protected Object
    readAtomicVector(VarNotes vi, TypeNotes ti, long count, List<Slice> slices)
            throws DapException
    {
        DapVariable atomvar = (DapVariable) getTemplate();
        DapType basetype = ti.getType();
        if(atomvar.getCount() == 0)
            return LibTypeFcns.newVector(basetype, 0);
        // Convert slices to (start,count,stride);
        int rank = atomvar.getRank();
        List<DapDimension> dimset = atomvar.getDimensions();
        Odometer odom = Odometer.factory(slices, dimset);
        List<Odometer> subodoms = odom.getSubOdometers();
        // Compute the total size of returned objects
        long totalsize = 0;
        for(int i = 0; i < subodoms.size(); i++) {
            Odometer ithodom = subodoms.get(i);
            totalsize += ithodom.totalSize();
        }
        Nc4prototypes nc4 = ((Nc4DSP) this.dsp).getJNI();
        SizeT[] startp = new SizeT[rank];
        SizeT[] countp = new SizeT[rank];
        SizeT[] stridep = new SizeT[rank];
        int ret;
        long edgecount;
        Odometer ithodom;
        Object partialresult;
        Object result = LibTypeFcns.newVector(basetype, totalsize);
        int pos = 0;
        for(int i = 0; i < subodoms.size(); i++) {
            ithodom = subodoms.get(i);
            edgecount = odomToEdges(ithodom, startp, countp, stridep);
            if(basetype.isFixedSize()) {
                long elemsize = ti.getSize();
                long memsize = edgecount * elemsize;
                Nc4Pointer mem = Nc4Pointer.allocate(memsize);
                readcheck(nc4, ret = nc4.nc_get_vars(vi.gid, vi.id, startp, countp, stridep, mem.p));
                partialresult = getatomicdata(ti.getType(), edgecount, elemsize, mem);
            } else if(basetype.isStringType()) {
                String[] ss = new String[(int) edgecount];
                readcheck(nc4, ret = nc4.nc_get_vars_string(vi.gid, vi.id, startp, countp, stridep, ss));
                partialresult = ss;
            } else if(basetype.isOpaqueType()) {
                long elemsize =  ti.getSize();
                edgecount = odomToEdges(ithodom, startp, countp, stridep);
                long memsize = edgecount * elemsize;
                Nc4Pointer mem = Nc4Pointer.allocate(memsize);
                    readcheck(nc4, ret = nc4.nc_get_vars(vi.gid, vi.id, startp, countp, stridep, mem.p));
                partialresult = new ByteBuffer[(int)edgecount];
                for(int ec=0;ec<edgecount;ec++) {
                    byte[] buf = mem.p.getByteArray(ec*ti.getSize(), (int) ti.getSize());
                    ((ByteBuffer[])partialresult)[ec] = ByteBuffer.wrap(buf);
                }
            } else
                throw new DapException("Unexpected atomic type: " + basetype);
            int len = Array.getLength(partialresult);
            System.arraycopy(partialresult, 0, result, pos, len);
            pos += len;
        }
        return result;
    }

    protected Nc4Cursor
    readStructure(Index index)
            throws DapException
    {
        assert (index != null);
        assert this.scheme == Scheme.STRUCTARRAY;
        DapVariable template = (DapVariable) getTemplate();
        VarNotes vi = (VarNotes) ((Nc4DSP) this.dsp).find(template);
        TypeNotes ti = vi.basetype;
        Nc4Pointer mem;
        Nc4Cursor cursor = null;
        if(template.isTopLevel()) {
            int ret;
            mem = Nc4Pointer.allocate(ti.getSize());
            Nc4prototypes nc4 = ((Nc4DSP) this.dsp).getJNI();
            if(index.getRank() == 0) {
                readcheck(nc4, ret = nc4.nc_get_var(vi.gid, vi.id, mem.p));
            } else {
                SizeT[] sizes = indexToSizes(index);
                readcheck(nc4, ret = nc4.nc_get_var1(vi.gid, vi.id, sizes, mem.p));
            }
            cursor = new Nc4Cursor(Scheme.STRUCTURE, (Nc4DSP) this.dsp, template, this);
        } else {// field of a structure instance or record
            long pos = index.index();
            if(pos < 0 || pos >= template.getCount())
                throw new IndexOutOfBoundsException("read: " + index);
            cursor = new Nc4Cursor(Scheme.STRUCTURE, (Nc4DSP) this.dsp, template, this);
            // Ok, we need to operate relative to the parent's memory
            // move to the appropriate offset
            mem = ((Nc4Cursor) getContainer()).getMemory().share(pos * ti.getSize(), ti.getSize());
        }
        cursor.setIndex(index);
        cursor.setMemory(mem);
        return cursor;
    }

    protected Nc4Cursor
    readSequence(Index index)
            throws DapException
    {
        assert (index != null);
        assert this.scheme == Scheme.SEQARRAY;
        DapVariable template = (DapVariable) getTemplate();
        VarNotes vi = (VarNotes) ((Nc4DSP) this.dsp).find(template);
        TypeNotes ti = vi.basetype;
        Nc4Pointer mem;
        Nc4Cursor cursor = null;
        Nc4prototypes.Vlen_t[] vlen = new Nc4prototypes.Vlen_t[1];
        // Given a seq var e.g v(d1,d2), where we have an index argument,
        // get that object, which will be a vlen
        if(template.isTopLevel()) {
            int ret;
            Nc4prototypes nc4 = ((Nc4DSP) this.dsp).getJNI();
            SizeT[] extents = indexToSizes(index);
            // read te index't vlen
            readcheck(nc4, ret = nc4.nc_get_var1(vi.gid, vi.id, extents, vlen));
        } else {// field of a structure instance or record
            long pos = index.index();
            if(pos < 0 || pos >= template.getCount())
                throw new IndexOutOfBoundsException("read: " + index);
            // The memory for a sequence field is the vector of vlen objects.
            // We need to extract the index'th vlen for this field
            Nc4Pointer pp = getMemory(); // should be the vector
            int vlensize = Nc4prototypes.Vlen_t.VLENSIZE;
            // point to the index'th element in the vector
            pp = pp.share(pos * vlensize, vlensize);
            // convert to a Vlen_T object
            vlen[0] = new Nc4prototypes.Vlen_t(pp.p);
            vlen[0].read();
        }
        // At this point, vlen[0] is the index'th vlen
        // Construct a sequence cursor whose memory is the memory of the vlen
        // (which is the vector of records)
        cursor = new Nc4Cursor(Scheme.SEQUENCE, (Nc4DSP) this.dsp, template, this);
        cursor.setRecordCount(vlen[0].len);
        long memsize = ti.getSize() * cursor.getRecordCount();
        mem = new Nc4Pointer(vlen[0].p, memsize);
        cursor.setMemory(mem);
        cursor.setIndex(index);
        return cursor;
    }

    //////////////////////////////////////////////////
    // Nc4Cursor Extensions

    public long
    getOffset()
    {
        DapVariable dv = (DapVariable) getTemplate();
        Notes n = ((Nc4DSP) this.dsp).find(dv);
        return n.getOffset();
    }

    public long
    getElementSize()
    {
        DapVariable dv = (DapVariable) getTemplate();
        Notes n = ((Nc4DSP) this.dsp).find(dv);
        return n.getSize();
    }

    public Nc4Pointer
    getMemory()
    {
        return this.memory;
    }

    public Nc4Cursor
    setMemory(Nc4Pointer p)
    {
        this.memory = p;
        return this;
    }

    //////////////////////////////////////////////////
    // Utilities

    protected long
    getElementSize(TypeNotes ti)
    {
        DapType type = ti.getType();
        switch (type.getTypeSort()) {
        case Structure:
        case Sequence:
            return ti.getSize();
        case String:
        case URL:
            return Pointer.SIZE;
        case Enum:
            return getElementSize((TypeNotes) ((Nc4DSP) getDSP()).find(ti.enumbase, NoteSort.TYPE));
        case Opaque:
            return ti.getSize();
        default:
            return type.getSize();
        }
    }

    protected Object
    getatomicdata(DapType basetype, long lcount, long elemsize, Nc4Pointer mem)
    {
        Object result = null;
        TypeSort sort = basetype.getTypeSort();
        int icount = (int) lcount;
        switch (sort) {
        case Char:
            // need to extract and convert utf8(really ascii) -> utf16
            byte[] bresult = mem.p.getByteArray(0, icount);
            char[] cresult = new char[bresult.length];
            for(int i = 0; i < icount; i++) {
                int ascii = bresult[i];
                ascii = ascii & 0x7F;
                cresult[i] = (char) ascii;
            }
            result = cresult;
            break;
        case UInt8:
        case Int8:
            result = mem.p.getByteArray(0, icount);
            break;
        case Int16:
        case UInt16:
            result = mem.p.getShortArray(0, icount);
            break;
        case Int32:
        case UInt32:
            result = mem.p.getIntArray(0, icount);
            break;
        case Int64:
        case UInt64:
            result = mem.p.getLongArray(0, icount);
            break;
        case Float32:
            result = mem.p.getFloatArray(0, icount);
            break;
        case Float64:
            result = mem.p.getDoubleArray(0, icount);
            break;
        case String:
        case URL:
            // TODO: properly free underlying strings
            result = mem.p.getStringArray(0, icount);
            break;
        case Opaque:
            ByteBuffer[] ops = new ByteBuffer[icount];
            result = ops;
            for(int i = 0; i < icount; i++) {
                ops[i] = mem.p.getByteBuffer(i * elemsize, elemsize);
            }
            break;
        case Enum:
            DapEnumeration de = (DapEnumeration) basetype;
            result = getatomicdata((DapType) de.getBaseType(), lcount, elemsize, mem);
            break;
        }
        return result;
    }

    static long
    odomToEdges(Odometer odom, SizeT[] startp, SizeT[] countp, SizeT[] stridep)
    {
        assert !odom.isMulti();
        int rank = odom.rank();
        List<Slice> slices = odom.getSlices();
        for(int i = 0; i < rank; i++) {
            Slice slice = slices.get(i);
            startp[i] = new SizeT(slice.getFirst());
            countp[i] = new SizeT(slice.getCount());
            stridep[i] = new SizeT(slice.getStride());
        }
        return DapUtil.sliceProduct(slices);
    }

    static public void
    errcheck(Nc4prototypes nc4, int ret)
            throws DapException
    {
        if(ret != 0) {
            String msg = String.format("Netcdf: errno=%d; %s", ret, nc4.nc_strerror(ret));
            throw new DapException(msg);
        }
    }

    static public void
    readcheck(Nc4prototypes nc4, int ret)
            throws DapException
    {
        try {
            errcheck(nc4, ret);
        } catch (DapException de) {
            throw new DapException(de);
        }
    }

    static SizeT[]
    indexToSizes(Index index)
    {
        SizeT[] sizes = new SizeT[index.getRank()];
        for(int i = 0; i < sizes.length; i++) {
            sizes[i] = new SizeT(index.get(i));
        }
        return sizes;
    }

    /**
     * Given a field ref, compute the true offset with respect to
     * it top-level containing structure/record
     *
     * @param f field cursor
     * @return
     * @throws DapException
     */
    long
    computeTrueOffset(Nc4Cursor f)
            throws DapException
    {
        List<Nc4Cursor> path = getCursorPath(f);
        long totaloffset = 0;
        Nc4Cursor current;

        // First element is presumed to be a structure ore record variable,
        // and that its memory covers only it's instance.
        // Walk intermediate nodes
        for(int i = 1; i < (path.size() - 1); i++) {
            current = path.get(i);
            DapVariable template = (DapVariable) current.getTemplate();
            VarNotes vi = (VarNotes) ((Nc4DSP) getDSP()).find(template);

            long size = vi.getSize();
            long offset = current.getOffset();
            long pos = 0;
            switch (current.getScheme()) {
            case SEQUENCE:
            case STRUCTURE:
                pos = current.getIndex().index();
                break;
            case RECORD:
                // readrecord will have set our memory to the start of the record
                pos = 0;
                break;
            default:
                throw new DapException("Illegal cursor type: " + current.getScheme());
            }
            long delta = size * pos + offset;
            totaloffset += delta;
        }
        assert path.get(path.size() - 1) == f;
        totaloffset += f.getOffset();
        return totaloffset;
    }

    /**
     * Given a cursor, get a list of "containing" cursors
     * with the following constraints.
     * 1. the first element in the path is a top-level variable.
     * 2. the remaining elements are the enclosing compound variables
     * 3. the last element is the incoming cursor.
     *
     * @param cursor
     * @return
     */
    static List<Nc4Cursor>
    getCursorPath(Nc4Cursor cursor)
    {
        List<Nc4Cursor> path = new ArrayList<>();
        for(; ; ) {
            if(!cursor.getScheme().isCompoundArray()) // suppress
                path.add(0, cursor);
            if(cursor.getScheme() == Scheme.SEQUENCE) {
                // Stop here because the sequence has the vlen mem as its mem
                break;
            }
            Nc4Cursor next = (Nc4Cursor) cursor.getContainer();
            if(next == null) {
                assert cursor.getTemplate().isTopLevel();
                break;
            }
            assert next.getTemplate().getSort() == DapSort.VARIABLE;
            cursor = next;
        }
        return path;
    }

    static Nc4Pointer
    getVarMemory(Nc4Cursor cursor)
    {
        while(cursor.getContainer() != null) {
            cursor = (Nc4Cursor) cursor.getContainer();
        }
        return cursor.getMemory();
    }

    /**
     * If the basetype is sequence (=> isVlen()),
     * then return the type of the first field of this sequence.
     * Otherwise return null.
     *
     * @return the type of the first field
     */
    public TypeNotes
    getVlenType(DapVariable v)
    {
        DapType t = v.getBaseType();
        if(t.getSort() != DapSort.SEQUENCE
                || ((DapSequence) t).getFields().size() != 1)
            throw new IllegalArgumentException(t.getFQN());
        DapSequence ds = (DapSequence) t;
        DapVariable f0 = ds.getField(0);
        DapType f0type = f0.getBaseType();
        return (TypeNotes) ((Nc4DSP) this.dsp).find(f0type);
    }

    protected void
    debug()
    {
        System.err.printf("CURSOR: %s%n", this.toString());
    }

}


package thredds.crawlabledataset.s3;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.google.common.base.Optional;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.io.Files;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A ThreddsS3Client that wraps another ThreddsS3Client and caches its methods' return values for efficiency.
 *
 * @author cwardgar
 * @since 2015/08/22
 */
public class CachingThreddsS3Client implements ThreddsS3Client {
    private static final Logger logger = LoggerFactory.getLogger(CachingThreddsS3Client.class);

    private static final long ENTRY_EXPIRATION_TIME = 1;  // In hours.
    private static final long MAX_METADATA_ENTRIES = 10000;
    private static final long MAX_FILE_ENTRIES = 100;

    private final ThreddsS3Client threddsS3Client;

    private final Cache<S3URI, Optional<ObjectMetadata>> objectMetadataCache;
    private final Cache<S3URI, Optional<ObjectListing>> objectListingCache;
    private final Cache<S3URI, Optional<File>> objectFileCache;

    public CachingThreddsS3Client(ThreddsS3Client threddsS3Client) {
        this(threddsS3Client, new ObjectFileCacheRemovalListener());
    }

    public CachingThreddsS3Client(
            ThreddsS3Client threddsS3Client, RemovalListener<S3URI, Optional<File>> removalListener) {
        this.threddsS3Client = threddsS3Client;

        // We can't reuse the builder because each of the caches we're creating has different type parameters.
        this.objectMetadataCache = CacheBuilder.newBuilder()
                .expireAfterAccess(ENTRY_EXPIRATION_TIME, TimeUnit.HOURS)
                .maximumSize(MAX_METADATA_ENTRIES)
                .build();
        this.objectListingCache = CacheBuilder.newBuilder()
                .expireAfterAccess(ENTRY_EXPIRATION_TIME, TimeUnit.HOURS)
                .maximumSize(MAX_METADATA_ENTRIES)
                .build();
        this.objectFileCache = CacheBuilder.newBuilder()
                .expireAfterAccess(ENTRY_EXPIRATION_TIME, TimeUnit.HOURS)
                .maximumSize(MAX_FILE_ENTRIES)
                .removalListener(removalListener)
                .build();
    }

    private static class ObjectFileCacheRemovalListener implements RemovalListener<S3URI, Optional<File>> {
        @Override
        public void onRemoval(RemovalNotification<S3URI, Optional<File>> notification) {
            Optional<File> file = notification.getValue();
            assert file != null : "Silence a silly IntelliJ warning. Of course the Optional isn't null.";

            if (file.isPresent()) {
                file.get().delete();
            }
        }
    }

    @Override
    public ObjectMetadata getObjectMetadata(S3URI s3uri) {
        Optional<ObjectMetadata> metadata = objectMetadataCache.getIfPresent(s3uri);
        if (metadata == null) {
            logger.debug(String.format("ObjectMetadata cache MISS: '%s'", s3uri));
            metadata = Optional.fromNullable(threddsS3Client.getObjectMetadata(s3uri));
            objectMetadataCache.put(s3uri, metadata);
        } else {
            logger.debug(String.format("ObjectMetadata cache hit: '%s'", s3uri));
        }

        return metadata.orNull();
    }

    @Override
    public ObjectListing listObjects(S3URI s3uri) {
        Optional<ObjectListing> objectListing = objectListingCache.getIfPresent(s3uri);
        if (objectListing == null) {
            logger.debug(String.format("ObjectListing cache MISS: '%s'", s3uri));
            objectListing = Optional.fromNullable(threddsS3Client.listObjects(s3uri));
            objectListingCache.put(s3uri, objectListing);
        } else {
            logger.debug(String.format("ObjectListing cache hit: '%s'", s3uri));
        }

        return objectListing.orNull();
    }

    /**
     * {@inheritDoc}
     *

     * WARNING: If the content at {@code s3uri} was previously saved using this method, and the old file to which it was
     * saved is not the same as {@code file}, the object content will be copied to the new file and the old file
     * will be deleted.
     */
    @Override
    public File saveObjectToFile(S3URI s3uri, File file) throws IOException {
        Optional<File> cachedFile = objectFileCache.getIfPresent(s3uri);

        if (cachedFile == null) {
            logger.debug("Object cache MISS: '%s'", s3uri);
            // Do download below.
        } else {
            logger.debug("Object cache hit: '%s'", s3uri);

            if (!cachedFile.isPresent()) {
                return null;
            } else if (!cachedFile.get().exists()) {
                logger.info(String.format("Found cache entry {'%s'-->'%s'}, but local file doesn't exist. " +
                        "Was it deleted? Re-downloading.", s3uri, cachedFile.get()));
                objectFileCache.invalidate(s3uri);  // Evict old entry. Re-download below.
            } else if (!cachedFile.get().equals(file)) {
                // Copy content of cachedFile to file. Evict cachedFile from the cache.
                Files.copy(cachedFile.get(), file);
                objectFileCache.put(s3uri, Optional.of(file));
                return file;
            } else {
                return file;  // File already contains the content of the object at s3uri.
            }
        }

        cachedFile = Optional.fromNullable(threddsS3Client.saveObjectToFile(s3uri, file));
        objectFileCache.put(s3uri, cachedFile);
        return cachedFile.orNull();
    }

    /**
     * Discards all entries from all caches. Any files created by {@link #saveObjectToFile} will be deleted.
     */
    public void clear() {
        objectMetadataCache.invalidateAll();
        objectListingCache.invalidateAll();
        objectFileCache.invalidateAll();
    }
}


package ucar.nc2.ui.util;

import javax.swing.*;
import java.awt.*;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.List;

/**
 * A collection of utility methods for Swing.
 *
 * @author Darryl Burke
 * @see "http://tips4java.wordpress.com/2008/11/13/swing-utils/"
 */
public final class SwingUtils {
    public static final Object NOT_NULL = new Object();

   private SwingUtils() {
      throw new Error("SwingUtils is just a container for static methods");
   }

   /**
    * Convenience method for searching below container in the
    * component hierarchy and return nested components that are instances of
    * class clazz it finds. Returns an empty list if no such
    * components exist in the container.
    *

    * Invoking this method with a class parameter of JComponent.class
    * will return all nested components.
    *

    * This method invokes getDescendantsOfType(clazz, container, true)
    *
    * @param clazz the class of components whose instances are to be found.
    * @param container the container at which to begin the search
    * @return the List of components
    */
   public static <T extends JComponent> List<T> getDescendantsOfType(
         Class<T> clazz, Container container) {
      return getDescendantsOfType(clazz, container, true);
   }

   /**
    * Convenience method for searching below container in the
    * component hierarchy and return nested components that are instances of
    * class clazz it finds. Returns an empty list if no such
    * components exist in the container.
    *

    * Invoking this method with a class parameter of JComponent.class
    * will return all nested components.
    *
    * @param clazz the class of components whose instances are to be found.
    * @param container the container at which to begin the search
    * @param nested true to list components nested within another listed
    * component, false otherwise
    * @return the List of components
    */
   public static <T extends JComponent> List<T> getDescendantsOfType(
         Class<T> clazz, Container container, boolean nested) {
      List<T> tList = new ArrayList<T>();
      for (Component component : container.getComponents()) {
         if (clazz.isAssignableFrom(component.getClass())) {
            tList.add(clazz.cast(component));
         }
         if (nested || !clazz.isAssignableFrom(component.getClass())) {
            tList.addAll(SwingUtils.<T>getDescendantsOfType(clazz,
                  (Container) component, nested));
         }
      }
      return tList;
   }

   /**
    * Convenience method that searches below container in the
    * component hierarchy and returns the first found component that is an
    * instance of class clazz having the bound property value.
    * Returns {@code null} if such component cannot be found.
    *

    * This method invokes getDescendantOfType(clazz, container, property, value,
    * true)
    *
    * @param clazz the class of component whose instance is to be found.
    * @param container the container at which to begin the search
    * @param property the className of the bound property, exactly as expressed in
    * the accessor e.g. "Text" for getText(), "Value" for getValue().
    * @param value the value of the bound property
    * @return the component, or null if no such component exists in the
    * container
    * @throws IllegalArgumentException if the bound property does
    * not exist for the class or cannot be accessed
    */
   public static <T extends JComponent> T getDescendantOfType(
         Class<T> clazz, Container container, String property, Object value)
         throws IllegalArgumentException {
      return getDescendantOfType(clazz, container, property, value, true);
   }

   /**
    * Convenience method that searches below container in the
    * component hierarchy and returns the first found component that is an
    * instance of class clazz and has the bound property value.
    * Returns {@code null} if such component cannot be found.
    *
    * @param clazz the class of component whose instance to be found.
    * @param container the container at which to begin the search
    * @param property the className of the bound property, exactly as expressed in
    * the accessor e.g. "Text" for getText(), "Value" for getValue().
    * @param value the value of the bound property
    * @param nested true to list components nested within another component
    * which is also an instance of clazz, false otherwise
    * @return the component, or null if no such component exists in the
    * container
    * @throws IllegalArgumentException if the bound property does
    * not exist for the class or cannot be accessed
    */
   public static <T extends JComponent> T getDescendantOfType(Class<T> clazz,
         Container container, String property, Object value, boolean nested)
         throws IllegalArgumentException {
      List<T> list = getDescendantsOfType(clazz, container, nested);
      return getComponentFromList(clazz, list, property, value);
   }

   /**
    * Convenience method for searching below container in the
    * component hierarchy and return nested components of class
    * clazz it finds.  Returns an empty list if no such
    * components exist in the container.
    *

    * This method invokes getDescendantsOfClass(clazz, container, true)
    *
    * @param clazz the class of components to be found.
    * @param container the container at which to begin the search
    * @return the List of components
    */
   public static <T extends JComponent> List<T> getDescendantsOfClass(
         Class<T> clazz, Container container) {
      return getDescendantsOfClass(clazz, container, true);
   }

   /**
    * Convenience method for searching below container in the
    * component hierarchy and return nested components of class
    * clazz it finds.  Returns an empty list if no such
    * components exist in the container.
    *
    * @param clazz the class of components to be found.
    * @param container the container at which to begin the search
    * @param nested true to list components nested within another listed
    * component, false otherwise
    * @return the List of components
    */
   public static <T extends JComponent> List<T> getDescendantsOfClass(
         Class<T> clazz, Container container, boolean nested) {
      List<T> tList = new ArrayList<T>();
      for (Component component : container.getComponents()) {
         if (clazz.equals(component.getClass())) {
            tList.add(clazz.cast(component));
         }
         if (nested || !clazz.equals(component.getClass())) {
            tList.addAll(SwingUtils.<T>getDescendantsOfClass(clazz,
                  (Container) component, nested));
         }
      }
      return tList;
   }

   /**
    * Convenience method that searches below container in the
    * component hierarchy in a depth first manner and returns the first
    * found component of class clazz having the bound property
    * value.
    *

    * Returns {@code null} if such component cannot be found.
    *

    * This method invokes getDescendantOfClass(clazz, container, property,
    * value, true)
    *
    * @param clazz the class of component to be found.
    * @param container the container at which to begin the search
    * @param property the className of the bound property, exactly as expressed in
    * the accessor e.g. "Text" for getText(), "Value" for getValue().
    * This parameter is case sensitive.
    * @param value the value of the bound property
    * @return the component, or null if no such component exists in the
    * container's hierarchy.
    * @throws IllegalArgumentException if the bound property does
    * not exist for the class or cannot be accessed
    */
   public static <T extends JComponent> T getDescendantOfClass(Class<T> clazz,
         Container container, String property, Object value)
         throws IllegalArgumentException {
      return getDescendantOfClass(clazz, container, property, value, true);
   }

   /**
    * Convenience method that searches below container in the
    * component hierarchy in a depth first manner and returns the first
    * found component of class clazz having the bound property
    * value.
    *

    * Returns {@code null} if such component cannot be found.
    *
    * @param clazz the class of component to be found.
    * @param container the container at which to begin the search
    * @param property the className of the bound property, exactly as expressed
    * in the accessor e.g. "Text" for getText(), "Value" for getValue().
    * This parameter is case sensitive.
    * @param value the value of the bound property
    * @param nested true to include components nested within another listed
    * component, false otherwise
    * @return the component, or null if no such component exists in the
    * container's hierarchy
    * @throws IllegalArgumentException if the bound property does
    * not exist for the class or cannot be accessed
    */
   public static <T extends JComponent> T getDescendantOfClass(Class<T> clazz,
         Container container, String property, Object value, boolean nested)
         throws IllegalArgumentException {
      List<T> list = getDescendantsOfClass(clazz, container, nested);
      return getComponentFromList(clazz, list, property, value);
   }

   private static <T extends JComponent> T getComponentFromList(Class<T> clazz,
         List<T> list, String property, Object value)
         throws IllegalArgumentException {
      T retVal = null;
      Method method = null;
      try {
         method = clazz.getMethod("get" + property);
      } catch (NoSuchMethodException ex) {
         try {
            method = clazz.getMethod("is" + property);
         } catch (NoSuchMethodException ex1) {
            throw new IllegalArgumentException("Property " + property +
                  " not found in class " + clazz.getName());
         }
      }
      try {
         for (T t : list) {
            Object testVal = method.invoke(t);
            if ((value == NOT_NULL && testVal != null) || (value != NOT_NULL && equals(value, testVal))) {
                return t;
            }
         }
      } catch (InvocationTargetException ex) {
         throw new IllegalArgumentException(
               "Error accessing property " + property +
               " in class " + clazz.getName());
      } catch (IllegalAccessException ex) {
         throw new IllegalArgumentException(
               "Property " + property +
               " cannot be accessed in class " + clazz.getName());
      } catch (SecurityException ex) {
         throw new IllegalArgumentException(
               "Property " + property +
               " cannot be accessed in class " + clazz.getName());
      }
      return retVal;
   }

   /**
    * Convenience method for determining whether two objects are either
    * equal or both null.
    *
    * @param obj1 the first reference object to compare.
    * @param obj2 the second reference object to compare.
    * @return true if obj1 and obj2 are equal or if both are null,
    * false otherwise
    */
   public static boolean equals(Object obj1, Object obj2) {
      return obj1 == null ? obj2 == null : obj1.equals(obj2);
   }

   /**
    * Convenience method for mapping a container in the hierarchy to its
    * contained components.  The keys are the containers, and the values
    * are lists of contained components.
    *

    * Implementation note:  The returned value is a HashMap and the values
    * are of type ArrayList.  This is subject to change, so callers should
    * code against the interfaces Map and List.
    *
    * @param container The JComponent to be mapped
    * @param nested true to drill down to nested containers, false otherwise
    * @return the Map of the UI
    */
   public static Map<JComponent, List<JComponent>> getComponentMap(
         JComponent container, boolean nested) {
      HashMap<JComponent, List<JComponent>> retVal =
            new HashMap<JComponent, List<JComponent>>();
      for (JComponent component : getDescendantsOfType(JComponent.class,
            container, false)) {
         if (!retVal.containsKey(container)) {
            retVal.put(container,
                  new ArrayList<JComponent>());
         }
         retVal.get(container).add(component);
         if (nested) {
            retVal.putAll(getComponentMap(component, nested));
         }
      }
      return retVal;
   }

   /**
    * Convenience method for retrieving a subset of the UIDefaults pertaining
    * to a particular class.
    *
    * @param clazz the class of interest
    * @return the UIDefaults of the class
    */
   public static UIDefaults getUIDefaultsOfClass(Class clazz) {
      String name = clazz.getName();
      name = name.substring(name.lastIndexOf(".") + 2);
      return getUIDefaultsOfClass(name);
   }

   /**
    * Convenience method for retrieving a subset of the UIDefaults pertaining
    * to a particular class.
    *
    * @param className fully qualified name of the class of interest
    * @return the UIDefaults of the class named
    */
   public static UIDefaults getUIDefaultsOfClass(String className) {
      UIDefaults retVal = new UIDefaults();
      UIDefaults defaults = UIManager.getLookAndFeelDefaults();
      List<?> listKeys = Collections.list(defaults.keys());
      for (Object key : listKeys) {
         if (key instanceof String && ((String) key).startsWith(className)) {
            String stringKey = (String) key;
            String property = stringKey;
            if (stringKey.contains(".")) {
               property = stringKey.substring(stringKey.indexOf(".") + 1);
            }
            retVal.put(property, defaults.get(key));
         }
      }
      return retVal;
   }

   /**
    * Convenience method for retrieving the UIDefault for a single property
    * of a particular class.
    *
    * @param clazz the class of interest
    * @param property the property to query
    * @return the UIDefault property, or null if not found
    */
   public static Object getUIDefaultOfClass(Class clazz, String property) {
      Object retVal = null;
      UIDefaults defaults = getUIDefaultsOfClass(clazz);
      List<Object> listKeys = Collections.list(defaults.keys());
      for (Object key : listKeys) {
         if (key.equals(property)) {
            return defaults.get(key);
         }
         if (key.toString().equalsIgnoreCase(property)) {
            retVal = defaults.get(key);
         }
      }
      return retVal;
   }

   /**
    * Exclude methods that return values that are meaningless to the user
    */
   static Set<String> setExclude = new HashSet<String>();
   static {
      setExclude.add("getFocusCycleRootAncestor");
      setExclude.add("getAccessibleContext");
      setExclude.add("getColorModel");
      setExclude.add("getGraphics");
      setExclude.add("getGraphicsConfiguration");
   }

   /**
    * Convenience method for obtaining most non-null human readable properties
    * of a JComponent.  Array properties are not included.
    *

    * Implementation note:  The returned value is a HashMap.  This is subject
    * to change, so callers should code against the interface Map.
    *
    * @param component the component whose proerties are to be determined
    * @return the class and value of the properties
    */
   public static Map<Object, Object> getProperties(JComponent component) {
      Map<Object, Object> retVal = new HashMap<Object, Object>();
      Class<?> clazz = component.getClass();
      Method[] methods = clazz.getMethods();
      Object value = null;
      for (Method method : methods) {
         if (method.getName().matches("^(is|get).*") &&
               method.getParameterTypes().length == 0) {
            try {
               Class returnType = method.getReturnType();
               if (returnType != void.class &&
                     !returnType.getName().startsWith("[") &&
                     !setExclude.contains(method.getName())) {
                  String key = method.getName();
                  value = method.invoke(component);
                  if (value != null && !(value instanceof Component)) {
                     retVal.put(key, value);
                  }
               }
            // ignore exceptions that arise if the property could not be accessed
            } catch (IllegalAccessException ex) {
            } catch (IllegalArgumentException ex) {
            } catch (InvocationTargetException ex) {
            }
         }
      }
      return retVal;
   }

   /**
    * Convenience method to obtain the Swing class from which this
    * component was directly or indirectly derived.
    *
    * @param component The component whose Swing superclass is to be
    * determined
    * @return The nearest Swing class in the inheritance tree
    */
   public static <T extends JComponent> Class getJClass(T component) {
      Class<?> clazz = component.getClass();
      while (!clazz.getName().matches("javax.swing.J[^.]*$")) {
         clazz = clazz.getSuperclass();
      }
      return clazz;
   }
}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.dap4lib.serial;

import dap4.core.data.ChecksumMode;
import dap4.core.dmr.*;
import dap4.core.util.*;
import dap4.dap4lib.LibTypeFcns;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;

import static dap4.core.data.DataCursor.Scheme;

public class D4DataCompiler
{
    static public boolean DEBUG = false;

    //////////////////////////////////////////////////
    // Constants

    static final public int COUNTSIZE = 8; // databuffer as specified by the DAP4 spec

    static String LBRACE = "{";
    static String RBRACE = "}";

    //////////////////////////////////////////////////
    // Instance variables

    protected DapDataset dataset = null;

    // Make compile arguments global
    protected ByteBuffer databuffer;

    protected ChecksumMode checksummode = null;
    protected ByteOrder order = null;

    protected D4DSP dsp;

    //////////////////////////////////////////////////
    //Constructor(s)

    /**
     * Constructor
     *
     * @param dsp          the D4DSP
     * @param checksummode
     * @param databuffer   the source of serialized databuffer
     */

    public D4DataCompiler(D4DSP dsp, ChecksumMode checksummode, ByteOrder order,
                          ByteBuffer databuffer)
            throws DapException
    {
        this.dsp = dsp;
        this.dataset = this.dsp.getDMR();
        this.databuffer = databuffer;
        this.checksummode = checksummode;
        this.order = order;
    }

    //////////////////////////////////////////////////
    // DataCompiler API

    /**
     * The goal here is to process the serialized
     * databuffer and locate top-level variable positions
     * in the serialized databuffer. Access to non-top-level
     * variables is accomplished on the fly.
     *
     * @return
     * @throws DapException
     */
    public void
    compile()
            throws DapException
    {
        assert (this.dataset != null && this.databuffer != null);
        // iterate over the variables represented in the databuffer
        for(DapVariable vv : this.dataset.getTopVariables()) {
            D4Cursor data = compileVar(vv, null);
            this.dsp.addVariableData(vv, data);
        }
    }

    protected D4Cursor
    compileVar(DapVariable dapvar, D4Cursor container)
            throws DapException
    {
        boolean isscalar = dapvar.getRank() == 0;
        D4Cursor array = null;
        DapType type = dapvar.getBaseType();
        if(type.isAtomic())
            array = compileAtomicVar(dapvar, container);
        else if(type.isStructType()) {
            array = compileStructureArray(dapvar, container);
        } else if(type.isSeqType()) {
            array = compileSequenceArray(dapvar, container);
        }
        if(dapvar.isTopLevel() && this.checksummode.enabled(ChecksumMode.DAP)) {
            // extract the checksum from databuffer src,
            // attach to the array, and make into an attribute
            int checksum = extractChecksum(databuffer);
            dapvar.setChecksum(checksum);
        }
        return array;
    }

    /**
     * @param var
     * @param container
     * @return
     * @throws DapException
     */

    protected D4Cursor
    compileAtomicVar(DapVariable var, D4Cursor container)
            throws DapException
    {
        DapType daptype = var.getBaseType();
        D4Cursor data = new D4Cursor(Scheme.ATOMIC, (D4DSP) this.dsp, var, container);
        data.setOffset(getPos(this.databuffer));
        long total = 0;
        long dimproduct = var.getCount();
        if(!daptype.isEnumType() && !daptype.isFixedSize()) {
            // this is a string, url, or opaque
            long[] positions = new long[(int) dimproduct];
            int savepos = databuffer.position();
            // Walk the bytestring and return the instance count (in databuffer)
            total = walkByteStrings(positions, databuffer);
            databuffer.position(savepos);// leave position unchanged
            data.setByteStringOffsets(total, positions);
        } else {
            total = dimproduct * daptype.getSize();
        }
        skip(databuffer, (int) total);
        return data;
    }

    /**
     * Compile a structure array.
     *
     * @param var    the template
     * @param container if inside a compound object
     * @return A DataCompoundArray for the databuffer for this struct.
     * @throws DapException
     */
    protected D4Cursor
    compileStructureArray(DapVariable var, D4Cursor container)
            throws DapException
    {
        DapStructure dapstruct = (DapStructure) var.getBaseType();
        D4Cursor structarray = new D4Cursor(Scheme.STRUCTARRAY, this.dsp, var, container)
                .setOffset(getPos(this.databuffer));
        List<DapDimension> dimset = var.getDimensions();
        long dimproduct = DapUtil.dimProduct(dimset);
        D4Cursor[] instances = new D4Cursor[(int) dimproduct];
        Odometer odom = Odometer.factory(DapUtil.dimsetToSlices(dimset), dimset);
        while(odom.hasNext()) {
            Index index = odom.next();
            D4Cursor instance = compileStructure(var, dapstruct, structarray);
            instance.setIndex(index);
            instances[(int) index.index()] = instance;
        }
        structarray.setElements(instances);
        return structarray;
    }

    /**
     * Compile a structure instance.
     *
     * @param dapstruct The template
     * @param container
     * @return A DataStructure for the databuffer for this struct.
     * @throws DapException
     */
    protected D4Cursor
    compileStructure(DapVariable var, DapStructure dapstruct, D4Cursor container)
            throws DapException
    {
        int pos = getPos(this.databuffer);
        D4Cursor d4ds = new D4Cursor(Scheme.STRUCTURE, (D4DSP) this.dsp, var, container)
                .setOffset(pos);
        List<DapVariable> dfields = dapstruct.getFields();
        for(int m = 0; m < dfields.size(); m++) {
            DapVariable dfield = dfields.get(m);
            D4Cursor dvfield = compileVar(dfield, d4ds);
            d4ds.addField(m, dvfield);
            assert dfield.getParent() != null;
        }
        return d4ds;
    }

    /**
     * Compile a sequence array.
     *
     * @param var the template
     * @return A DataCompoundArray for the databuffer for this sequence.
     * @throws DapException
     */
    protected D4Cursor
    compileSequenceArray(DapVariable var, D4Cursor container)
            throws DapException
    {
        DapSequence dapseq = (DapSequence) var.getBaseType();
        D4Cursor seqarray = new D4Cursor(Scheme.SEQARRAY, this.dsp, var, container)
                .setOffset(getPos(this.databuffer));
        List<DapDimension> dimset = var.getDimensions();
        long dimproduct = DapUtil.dimProduct(dimset);
        D4Cursor[] instances = new D4Cursor[(int) dimproduct];
        Odometer odom = Odometer.factory(DapUtil.dimsetToSlices(dimset), dimset);
        while(odom.hasNext()) {
            Index index = odom.next();
            D4Cursor instance = compileSequence(var, dapseq, seqarray);
            instance.setIndex(index);
            instances[(int) index.index()] = instance;
        }
        seqarray.setElements(instances);
        return seqarray;
    }

    /**
     * Compile a sequence as a set of records.
     *
     * @param dapseq
     * @param container
     * @return
     * @throws DapException
     */
    public D4Cursor
    compileSequence(DapVariable var, DapSequence dapseq, D4Cursor container)
            throws DapException
    {
        int pos = getPos(this.databuffer);
        D4Cursor seq = new D4Cursor(Scheme.SEQUENCE, this.dsp, var, container)
                .setOffset(pos);
        List<DapVariable> dfields = dapseq.getFields();
        // Get the count of the number of records
        long nrecs = getCount(this.databuffer);
        for(int r = 0; r < nrecs; r++) {
            pos = getPos(this.databuffer);
            D4Cursor rec = (D4Cursor) new D4Cursor(D4Cursor.Scheme.RECORD, this.dsp, var, container)
                    .setOffset(pos).setRecordIndex(r);
            for(int m = 0; m < dfields.size(); m++) {
                DapVariable dfield = dfields.get(m);
                D4Cursor dvfield = compileVar(dfield, rec);
                rec.addField(m, dvfield);
                assert dfield.getParent() != null;
            }
            seq.addRecord(rec);
        }
        return seq;
    }

    //////////////////////////////////////////////////
    // Utilities

    protected int
    extractChecksum(ByteBuffer data)
            throws DapException
    {
        assert ChecksumMode.DAP.enabled(this.checksummode);
        if(data.remaining() < DapUtil.CHECKSUMSIZE)
            throw new DapException("Short serialization: missing checksum");
        return data.getInt();
    }

    static protected void
    skip(ByteBuffer data, int count)
    {
        data.position(data.position() + count);
    }

    static protected int
    getCount(ByteBuffer data)
    {
        long count = data.getLong();
        count = (count & 0xFFFFFFFF);
        return (int) count;
    }

    static protected int
    getPos(ByteBuffer data)
    {
        return data.position();
    }

    /**
     * Compute the size in databuffer of the serialized form
     *
     * @param daptype
     * @return type's serialized form size
     */
    static protected int
    computeTypeSize(DapType daptype)
    {
        return LibTypeFcns.size(daptype);
    }

    static protected long
    walkByteStrings(long[] positions, ByteBuffer databuffer)
    {
        int count = positions.length;
        long total = 0;
        int savepos = databuffer.position();
        // Walk each bytestring
        for(int i = 0; i < count; i++) {
            int pos = databuffer.position();
            positions[i] = pos;
            int size = getCount(databuffer);
            total += COUNTSIZE;
            total += size;
            skip(databuffer, size);
        }
        databuffer.position(savepos);// leave position unchanged
        return total;
    }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.unidata.io;

import java.io.IOException;
import java.io.InputStream;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.BufferedInputStream;
import java.net.URL;

/**
 * Big-endian little-endian data input stream, from which numbers in
 * both big- and little-endian representations can be read.  This is
 * needed for ESRI shapefiles, for example, since they contain both
 * big- and little-endian representations.
 *
 * @author Russ Rew
 */
public class BeLeDataInputStream extends DataInputStream {
  private byte w[] = new byte[8]; // work array for buffering bytes
  private static final int kLongs = 128; // arbitrary, probably big enough
  private long[] longWorkSpace = new long[kLongs];
  private byte[] byteWorkSpace = new byte[8 * kLongs];

  /**
   * Construct a bigEndian-littleEndian input stream from an input stream.
   *
   * @param inputStream from which to read
   */
  public BeLeDataInputStream(InputStream inputStream) throws IOException {
    super(inputStream);
  }

  /**
   * Construct a bigEndian-littleEndian input stream from a file.
   *
   * @param filename of file
   */
  public BeLeDataInputStream(String filename) throws IOException {
    this(new BufferedInputStream(new FileInputStream(filename)));
  }

  /**
   * Construct a bigEndian-littleEndian input stream from a URL.
   *
   * @param url of remote data
   */
  public BeLeDataInputStream(URL url) throws IOException {
    this(new BufferedInputStream(new DataInputStream(url.openStream())));
  }

  /**
   * read an int in little endian format
   *
   * @return int created from next 4 bytes in stream, in littleEndian order
   */
  public int readLEInt()
          throws IOException {
    readFully(w, 0, 4);
    return (w[3] & 0xff) << 24 |
            (w[2] & 0xff) << 16 |
            (w[1] & 0xff) << 8 |
            (w[0] & 0xff);
  }

  /**
   * read a double in little endian format
   *
   * @return double from next 8 bytes in stream, littleEndian order
   */
  public double readLEDouble() throws IOException {
    return Double.longBitsToDouble(readLELong());
  }

  /**
   * Reads n little-endian doubles from a random access file.
   *

   *
 This method is provided for speed when accessing a number
   * of consecutive values of the same type.
   *
   * @param d the buffer into which the doubles are read
   * @param n number of little-endian doubles to read
   */
  public final void readLEDoubles(double[] d, int n) throws IOException {
    int nLeft = n;
    int dCount = 0;
    int nToRead = kLongs;
    while (nLeft > 0) {
      if (nToRead > nLeft)
        nToRead = nLeft;
      readLELongs(longWorkSpace, nToRead);
      for (int i = 0; i < nToRead; i++) {
        d[dCount++] = Double.longBitsToDouble(longWorkSpace[i]);
      }
      nLeft -= nToRead;
    }
  }

  /**
   * read a long in little endian format
   *
   * @return long from next 8 bytes in stream, littleEndian order
   */
  public long readLELong() throws IOException {
    readFully(w, 0, 8);
    return
            (long) (w[7] & 0xff) << 56 |
                    (long) (w[6] & 0xff) << 48 |
                    (long) (w[5] & 0xff) << 40 |
                    (long) (w[4] & 0xff) << 32 |
                    (long) (w[3] & 0xff) << 24 |
                    (long) (w[2] & 0xff) << 16 |
                    (long) (w[1] & 0xff) << 8 |
                    (long) (w[0] & 0xff);
  }

  /**
   * Reads n little-endian longs from a random access file.
   *

   *
 This method is provided for speed when accessing a number
   * of consecutive values of the same type.
   *
   * @param lbuf the buffer into which the longs are read
   * @param n    the number of little-endian longs to read
   */
  public final void readLELongs(long lbuf[], int n) throws IOException {
    int nLeft = n;
    int lCount = 0;
    int nToRead = kLongs;
    while (nLeft > 0) {
      if (nToRead > nLeft)
        nToRead = nLeft;
      readFully(byteWorkSpace, 0, 8 * nToRead);
      int j = 0;
      for (int i = 0; i < nToRead; i++) {
        lbuf[lCount++] =
                (long) (byteWorkSpace[j] & 0xff) |
                        (long) (byteWorkSpace[j + 1] & 0xff) << 8 |
                        (long) (byteWorkSpace[j + 2] & 0xff) << 16 |
                        (long) (byteWorkSpace[j + 3] & 0xff) << 24 |
                        (long) (byteWorkSpace[j + 4] & 0xff) << 32 |
                        (long) (byteWorkSpace[j + 5] & 0xff) << 40 |
                        (long) (byteWorkSpace[j + 6] & 0xff) << 48 |
                        (long) (byteWorkSpace[j + 7] & 0xff) << 56;
        j += 8;
      }
      nLeft -= nToRead;
    }
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.point.standard;

import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.Dimension;
import ucar.nc2.Variable;
import ucar.nc2.Attribute;
import ucar.nc2.constants.AxisType;
import ucar.nc2.constants.CF;

/**
 * Abstract superclass for TableConfigurer implementations
 * @author caron
 * @since Jan 21, 2009
 */
public abstract class TableConfigurerImpl implements TableConfigurer {

  @Override
  public String getConvName() {
    return convName;
  }

  @Override
  public void setConvName(String convName) {
    this.convName = convName;
  }

  @Override
  public String getConvUsed() {
    return convUsed;
  }

  @Override
  public void setConvUsed(String convUsed) {
    this.convUsed = convUsed;
  }

  private String convName, convUsed;

  /* protected Variable findVariableWithStandardNameAndNotDimension(NetcdfDataset ds, String standard_name, Dimension outer, Formatter errlog) {
    for (Variable v : ds.getVariables()) {
      String stdName = ds.findAttValueIgnoreCase(v, CF.STANDARD_NAME, null);
      if ((stdName != null) && stdName.equals(standard_name) && v.getRank() > 0 && !v.getDimension(0).equals(outer))
        return v;
    }
    return null;
  }

  protected String matchAxisTypeAndDimension(NetcdfDataset ds, AxisType type, final Dimension outer) {
    Variable var = CoordSysEvaluator.findCoordByType(ds, type, new CoordSysEvaluator.Predicate() {
      public boolean match(CoordinateAxis axis) {
        if ((outer == null) && (axis.getRank() == 0))
          return true;
        if ((outer != null) && (axis.getRank() == 1) && (outer.equals(axis.getDimension(0))))
          return true;
        return false;
      }
    });
    if (var == null) return null;
    return var.getShortName();
  } */

  protected String matchAxisTypeAndDimension(NetcdfDataset ds, AxisType type, final Dimension outer, final Dimension inner) {
    Variable var = CoordSysEvaluator.findCoordByType(ds, type, new CoordSysEvaluator.Predicate() {
      @Override
      public boolean match(CoordinateAxis axis) {
        return ((axis.getRank() == 2) && outer.equals(axis.getDimension(0)) && inner.equals(axis.getDimension(1)));
      }
    });
    if (var == null) return null;
    return var.getShortName();
  }

  protected String matchAxisTypeAndDimension(NetcdfDataset ds, AxisType type, final Dimension outer, final Dimension middle, final Dimension inner) {
    Variable var = CoordSysEvaluator.findCoordByType(ds, type, new CoordSysEvaluator.Predicate() {
      @Override
      public boolean match(CoordinateAxis axis) {
        return ((axis.getRank() == 3) && outer.equals(axis.getDimension(0)) && middle.equals(axis.getDimension(1)) && inner.equals(axis.getDimension(2)));
      }
    });
    if (var == null) return null;
    return var.getShortName();
  }

  protected CoordinateAxis findZAxisNotStationAlt(NetcdfDataset ds) {
    CoordinateAxis z = CoordSysEvaluator.findCoordByType(ds, AxisType.Height, new NotStationAlt());
    if (z != null) return z;

    z = CoordSysEvaluator.findCoordByType(ds, AxisType.Pressure, new NotStationAlt());
    if (z != null) return z;

    z = CoordSysEvaluator.findCoordByType(ds, AxisType.GeoZ, new NotStationAlt());
    return z;
  }

  // search for an axis which is  not the station altitude
  private static class NotStationAlt implements CoordSysEvaluator.Predicate {

    @Override
    public boolean match(CoordinateAxis axis) {
      Attribute stdName = axis.findAttribute(CF.STANDARD_NAME);
       if (stdName == null) return true;
       String val = stdName.getStringValue();
       return !CF.SURFACE_ALTITUDE.equals(val) && !CF.STATION_ALTITUDE.equals(val);
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import javax.annotation.Nullable;
import thredds.featurecollection.FeatureCollectionConfig;
import ucar.nc2.grib.GdsHorizCoordSys;
import ucar.nc2.grib.GribTables;
import ucar.nc2.grib.grib1.Grib1Gds;
import ucar.nc2.grib.grib1.Grib1SectionGridDefinition;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib1.tables.Grib1ParamTables;
import ucar.unidata.io.RandomAccessFile;

import java.io.IOException;

/**
 * Grib1-specific reading of ncx files.
 *
 * @author caron
 * @since 2/20/14
 */
public class Grib1CollectionBuilderFromIndex extends GribCollectionBuilderFromIndex {

  // read in the index, index raf already open; return null on failure
  @Nullable
  static Grib1Collection readFromIndex(String name, RandomAccessFile raf,
      FeatureCollectionConfig config, org.slf4j.Logger logger) {

    Grib1CollectionBuilderFromIndex builder = new Grib1CollectionBuilderFromIndex(name, config, logger);
    if (!builder.readIndex(raf))
      return null;

    if (builder.gc.getFiles().size() == 0) {
      logger.warn("Grib1CollectionBuilderFromIndex {}: has no files, force recreate ", builder.gc.getName());
      return null;
    }

    return new Grib1Collection(builder.gc);
  }

  // read in the index, index raf already open; return null on failure
  @Nullable
  static GribCollectionMutable openMutableGCFromIndex(String name, RandomAccessFile raf, FeatureCollectionConfig config, org.slf4j.Logger logger) {

    Grib1CollectionBuilderFromIndex builder = new Grib1CollectionBuilderFromIndex(name, config, logger);
    if (!builder.readIndex(raf))
      return null;

    if (builder.gc.getFiles().size() == 0) {
      logger.warn("Grib1CollectionBuilderFromIndex {}: has no files, force recreate ", builder.gc.getName());
      return null;
    }

    return builder.gc;
  }

 ////////////////////////////////////////////////////////////////

  protected Grib1Customizer cust; // gets created in readIndex, after center etc is read in

  Grib1CollectionBuilderFromIndex(String name, FeatureCollectionConfig config,
      org.slf4j.Logger logger) {
    super( new GribCollectionMutable(name, null, config, true), config, logger);  // directory will be set in readFromIndex
  }

  protected int getVersion() {
    return Grib1CollectionWriter.version;
  }
  protected int getMinVersion() {
    return Grib1CollectionWriter.minVersion;
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////
  // reading

  protected String getMagicStart() {
    return Grib1CollectionWriter.MAGIC_START;
  }

  protected GribTables makeCustomizer()  throws IOException {
    Grib1ParamTables ptables = (config.gribConfig.paramTable != null) ? Grib1ParamTables.factory(config.gribConfig.paramTable) :
            Grib1ParamTables.factory(config.gribConfig.paramTablePath, config.gribConfig.lookupTablePath); // so an iosp message must be received before the open()
    this.cust = Grib1Customizer.factory(gc.center, gc.subcenter, gc.version, ptables);
    return cust;
  }

  protected String getLevelNameShort(int levelCode) {
    return cust.getLevelNameShort(levelCode);
  }

  @Override
  protected GribHorizCoordSystem readGds(GribCollectionProto.Gds p) {
    byte[] rawGds = null;
    Grib1Gds gds;
    int predefined = -1;
    if (p.getPredefinedGridDefinition() > 0) {
      predefined = p.getPredefinedGridDefinition();
      gds = ucar.nc2.grib.grib1.Grib1GdsPredefined.factory(gc.center, predefined);
    } else {
      rawGds = p.getGds().toByteArray();
      Grib1SectionGridDefinition gdss = new Grib1SectionGridDefinition(rawGds);
      gds = gdss.getGDS();
    }

    GdsHorizCoordSys hcs = gds.makeHorizCoordSys();
    String hcsName = (hcs == null) ? gds.getClass().getName() :  makeHorizCoordSysName(hcs);

    // check for user defined group names
    String desc = null;
    if (config.gribConfig.gdsNamer != null)
      desc = config.gribConfig.gdsNamer.get(gds.hashCode());
    if (desc == null)
      desc = (hcs == null) ? hcsName : hcs.makeDescription(); // default desc

    return new GribHorizCoordSystem(hcs, rawGds, gds, hcsName, desc, predefined);
  }

}