/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information. */

package dap4.core.util;

import dap4.core.dmr.*;

import java.io.*;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;

/**
 * Misc. Utility methods
 */

abstract public class DapUtil // Should only contain static methods
{
    //////////////////////////////////////////////////
    // Constants

    static final public BigInteger BIG_UMASK64 = new BigInteger("FFFFFFFFFFFFFFFF", 16);
    static final public Charset UTF8 = Charset.forName("UTF-8");

    // Define the Serialization Constants common to servlet and client

    static final public ByteOrder NETWORK_ORDER = ByteOrder.BIG_ENDIAN;
    static final public ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder();

    // Use Bit flags to avoid heavyweight enumset
    static public final int CHUNK_DATA = 0;  // bit 0 : value 0
    static final public int CHUNK_END = 1;   // bit 0 : value 1
    static public final int CHUNK_ERROR = 2; // bit 1 : value 1
    static public final int CHUNK_LITTLE_ENDIAN = 4; // bit 2: value 1
    static public final int CHUNK_NOCHECKSUM = 8; // bit 2: value 1

    // Construct the union of all flags
    static final public int CHUNK_ALL
            = CHUNK_DATA | CHUNK_ERROR | CHUNK_END | CHUNK_LITTLE_ENDIAN;

    static final public String LF = "\n";
    static final public String CRLF = "\r\n";
    static final public int CRLFSIZE = 2;

    //static final public int CHECKSUMSIZE = 16; // bytes if MD5
    //static final public String DIGESTOR = "MD5";

    static final public int CHECKSUMSIZE = 4; // bytes if CRC32
    static final public String DIGESTER = "CRC32";
    static final public String CHECKSUMATTRNAME = "_DAP4_Checksum_CRC32";
    static final public String LITTLEENDIANATTRNAME = "_DAP4_Little_Endian";
    static final public String CEATTRNAME = "_dap4.ce";

    static final public String DRIVELETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

    //////////////////////////////////////////////////
    // return last name part of an fqn; result will be escaped.

    static public String fqnSuffix(String fqn)
    {
        int structindex = fqn.lastIndexOf('.');
        int groupindex = fqn.lastIndexOf('/');
        if(structindex >= 0)
            return fqn.substring(structindex + 1, fqn.length());
        else
            return fqn.substring(groupindex + 1, fqn.length());
    }

    // return prefix name part of an fqn; result will be escaped.
    static public String fqnPrefix(String fqn)
    {
        int structindex = fqn.lastIndexOf('.');
        int groupindex = fqn.lastIndexOf('/');
        if(structindex >= 0)
            return fqn.substring(0, structindex);
        else
            return fqn.substring(0, groupindex);
    }

    /**
     * Split a string with respect to a separator
     * character and taking backslashes into consideration.
     *
     * @param s   The string to split
     * @param sep The character on which to split
     * @return a List of strings (all with escaping still intact)
     * representing s split at unescaped instances of sep.
     */
    static public List<String>
    backslashSplit(String s, char sep)
    {
        List<String> path = new ArrayList<String>();
        int len = s.length();
        StringBuilder piece = new StringBuilder();
        int i = 0;
        for(; i <= len - 1; i++) {
            char c = s.charAt(i);
            if(c == '\\' && i < (len - 1)) {
                piece.append(c); // keep escapes in place
                piece.append(s.charAt(++i));
            } else if(c == sep) {
                path.add(piece.toString());
                piece.setLength(0);
            } else
                piece.append(c);
        }
        path.add(piece.toString());
        return path;
    }

    static public Integer
    stringToInteger(String s)
    {
        try {
            return Integer.parseInt(s);
        } catch (NumberFormatException e) {
            return null;
        }
    }

    static public boolean hasSequence(DapNode node)
    {
        switch (node.getSort()) {
        case SEQUENCE:
            return true;

        case STRUCTURE:
            DapStructure container = (DapStructure) node;
            for(int i = 0; i < container.getFields().size(); i++) {
                if(hasSequence(container.getFields().get(i)))
                    return true;
            }
            break;

        case GROUP:
            DapGroup group = (DapGroup) node;
            for(int i = 0; i < group.getDecls().size(); i++) {
                if(hasSequence(group.getDecls().get(i)))
                    return true;
            }
            break;

        default: /* ignore */
            break;
        }
        return false;
    }

    /**
     * Walk the specified subtree dir tree to try to locate file|dir named filename.
     * Use breadth first search.
     *
     * @param filename Name of the file|dir to locate
     * @param abspath  Absolute path from which to start search
     * @param wantdir  True if we are looking for a directory,
     *                 false if we are looking for a file
     * @return absolute path of the file or null
     */
    static public String
    locateFile(String filename, String abspath, boolean wantdir)
    {
        Deque<String> q = new ArrayDeque<String>();
        // clean up the path and filename
        filename = filename.trim().replace('\\', '/');
        abspath = abspath.trim().replace('\\', '/');
        if(filename.charAt(0) == '/') filename = filename.substring(1);
        if(filename.endsWith("/")) filename = filename.substring(0, filename.length() - 1);
        if(abspath.endsWith("/")) abspath = abspath.substring(0, abspath.length() - 1);
        q.addFirst(abspath);  // prime the search queue

        for(; ; ) {  // breadth first search
            String currentpath = q.poll();
            if(currentpath == null) break; // done searching
            File current = new File(currentpath);
            File[] contents = current.listFiles();
            if(contents != null) {
                for(File subfile : contents) {
                    if(!subfile.getName().equals(filename)) continue;
                    if((wantdir && subfile.isDirectory())
                            || (!wantdir && subfile.isFile())) {
                        // Assume this is it
                        return DapUtil.canonicalpath(subfile.getAbsolutePath());
                    }
                }
                for(File subfile : contents) {
                    if(subfile.isDirectory())
                        q.addFirst(currentpath + "/" + subfile.getName());
                }
            }
        }
        return null;
    }

    /**
     * Walk the specified dir tree to locate file specified by relative path.
     * Use breadth first search.
     *
     * @param relpath Name of the file|dir to locate
     * @param abspath Absolute path from which to start search
     * @param wantdir True if we are looking for a directory,
     *                false if we are looking for a file
     * @return absolute path of the file|dir wrt to abspath
     */
    static public String
    locateRelative(String relpath, String abspath, boolean wantdir)
    {
        // clean up the path and filename
        relpath = relpath.trim().replace('\\', '/');
        if(relpath.charAt(0) == '/') relpath = relpath.substring(1);
        if(relpath.endsWith("/")) relpath = relpath.substring(0, relpath.length() - 1);
        String[] pieces = relpath.split("[/]");
        String partial = abspath;
        for(int i = 0; i < pieces.length - 1; i++) {
            String nextdir = locateFile(pieces[i], abspath, true);
            if(nextdir == null) return null;
            partial = nextdir;
        }
        // See if the final file|dir exists in this dir
        String finalpath = locateFile(pieces[pieces.length - 1], partial, wantdir);
        return finalpath;
    }

    /**
     * Convert path to:
     * 1. use '/' consistently
     * 2. remove any trailing '/'
     * 3. trim blanks
     * 4. Be aware of possible windows drive letter
     *
     * @param path convert this path
     * @return canonicalized version
     */
    static public String
    canonicalpath(String path)
    {
        if(path == null) return null;
        path = path.trim();
        path = path.replace('\\', '/');
        if(path.endsWith("/"))
            path = path.substring(0, path.length() - 1);
        boolean abs = (path.length() > 0 && path.charAt(0) == '/');
        if(abs) path = path.substring(1); // temporary
        if(DapUtil.hasDriveLetter(path)) {
            // As a last step, lowercase the drive letter, if any
            path = path.substring(0, 1).toLowerCase() + path.substring(1);
        } else if(abs)
            path = "/" + path;
        return path;
    }

    static public String
    relativize(String path)
    {
        if(path != null) {
            if(path.startsWith("/"))
                path = path.substring(1);
            if(hasDriveLetter(path))
                path = path.substring(2);
        }
        return path;
    }

    static public String
    absolutize(String path)
    {
        if(path != null && !path.startsWith("/") && !hasDriveLetter(path))
            path = "/" + path;
        return path;
    }

    static public boolean
    checkFixedSize(DapVariable var)
    {
        DapType dt = var.getBaseType();
        switch (dt.getTypeSort()) {
        case Structure:
        case Sequence:
            for(DapVariable field : ((DapStructure) dt).getFields()) {
                if(!checkFixedSize(field)) return false;
            }
            break;

        default:
            return dt.isFixedSize();
        }
        return true;
    }

    /**
     * Properly extract the byte contents of a ByteBuffer
     *
     * @param buf The buffer whose content is to be extracted
     *            as defined by the buffer limit.
     * @return The byte array contents of the buffer
     */
    static public byte[] extract(ByteBuffer buf)
    {
        int len = buf.limit();
        byte[] bytes = new byte[len];
        buf.rewind();
        buf.get(bytes);
        return bytes;
    }

    static public byte[]
    readbinaryfile(InputStream stream)
            throws IOException
    {
        // Extract the stream into a bytebuffer
        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
        byte[] tmp = new byte[1 << 16];
        for(; ; ) {
            int cnt;
            try {
                cnt = stream.read(tmp);
                if(cnt <= 0) break;
                bytes.write(tmp, 0, cnt);
            } catch (IOException ioe) {
                throw new dap4.core.util.DapException(ioe);
            }
        }
        return bytes.toByteArray();
    }

    static public String
    readtextfile(InputStream stream)
            throws IOException
    {
        StringBuilder buf = new StringBuilder();
        InputStreamReader rdr = new InputStreamReader(stream, UTF8);
        for(; ; ) {
            int c = rdr.read();
            if(c < 0) break;
            buf.append((char) c);
        }
        return buf.toString();
    }

    /**
     * Given a dap variable, get the path  from the
     * top-level variable to and including the given variable
     * such that all but the last element is a structure.
     */
    static public List<DapVariable>
    getStructurePath(DapVariable var)
    {
        List<DapNode> path = var.getPath();
        List<DapVariable> structpath = new ArrayList<DapVariable>();
        for(int i = 0; i < path.size(); i++) {
            DapNode node = path.get(i);
            switch (node.getSort()) {
            case DATASET:
            case GROUP:
                break;
            case VARIABLE:
                structpath.add((DapVariable) node);
                break;
            default:
                assert false : "Internal error";
            }
        }
        return structpath;
    }

    /**
     * Convert null paths to ""
     *
     * @param path
     * @return path or ""
     */
    static public String
    denullify(String path)
    {
        return (path == null ? "" : path);
    }

    /**
     * Convert "" paths to null
     *
     * @param path
     * @return path or null
     */
    static public String
    nullify(String path)
    {
        return (path != null && path.length() == 0 ? null : path);
    }

    static public long[]
    getDimSizes(List<DapDimension> dims)
    {
        long[] sizes = new long[dims.size()];
        for(int i = 0; i < dims.size(); i++) {
            sizes[i] = dims.get(i).getSize();
        }
        return sizes;
    }

    static public long dimProduct(List<DapDimension> dimset) // dimension crossproduct
    {
        long count = 1;
        for(DapDimension dim : dimset) {
            count *= dim.getSize();
        }
        return count;
    }

    /**
     * Given a view, get the universal
     * View either from a DapDataset.
     */
/*
    static public View
    createView(DapDataset dataset)
        throws DapException
    {
        View u = new View(dataset);
        List<DapNode> nodes = dataset.getNodeList();
        for(DapNode node : nodes) {
            switch (node.getSort()) {
            case VARIABLE:
                u.put((DapVariable) node);
                break;
            default:
                break;
            }
        }
        u.finish(!View.EXPAND);
        return u;
    }
*/

    /**
     * Test a List<Slice> against set of DapDimensions
     * to see if the list is whole wrt the dimensions
     *
     * @param slices the set of slices
     * @param dimset the list of DapDimension
     * @result true if slices is whole wrt dimset; false otherwise
     */
    static public boolean
    isWhole(List<Slice> slices, List<DapDimension> dimset)
    {
        if(slices.size() != dimset.size())
            return false;
        for(int i = 0; i < slices.size(); i++) {
            Slice slice = slices.get(i);
            DapDimension dim = dimset.get(i);
            if(slice.getStride() != 1 || slice.getFirst() != 0 || slice.getCount() != dim.getSize())
                return false;
        }
        return true;
    }

    static public long
    sliceProduct(List<Slice> slices) // another crossproduct
    {
        long count = 1;
        if(slices != null)
            for(Slice slice : slices) {
                count *= slice.getCount();
            }
        return count;
    }

    static public boolean
    hasStrideOne(List<Slice> slices)
    {
        for(Slice slice : slices) {
            if(slice.getStride() != 1)
                return false;
        }
        return true;
    }

    /**
     * Given an Array of Strings and a separator and a count,
     * concat the first count elements of an array with separator
     * between them. A null string is treated like "".
     *
     * @param array the array to concat
     * @param sep   the separator
     * @param from  start point for join (inclusive)
     * @param upto  end point for join (exclusive)
     * @return the join
     */
    static public String
    join(String[] array, String sep, int from, int upto)
    {
        if(sep == null) sep = "";
        if(from < 0 || upto > array.length)
            throw new IndexOutOfBoundsException();
        if(upto <= from)
            return "";
        StringBuilder result = new StringBuilder();
        boolean first = true;
        for(int i = from; i < upto; i++, first = false) {
            if(!first) result.append(sep);
            result.append(array[i]);
        }
        return result.toString();
    }

    /**
     * Relativizing a path =>  remove any leading '/' and cleaning it
     *
     * @param path
     * @return
     */
    static public String
    xrelpath(String path)
    {
        return DapUtil.canonicalpath(path);
    }

    /**
     * return true if this path appears to start with a windows drive letter
     *
     * @param path
     * @return
     */
    static public boolean
    hasDriveLetter(String path)
    {
        boolean hasdr = false;
        if(path != null && path.length() >= 2) {
            hasdr = (DRIVELETTERS.indexOf(path.charAt(0)) >= 0 && path.charAt(1) == ':');
        }
        return hasdr;
    }

    /**
     * Return the set of leading protocols for a url; may be more than one.
     *
     * @param url        the url whose protocols to return
     * @param breakpoint return the index past last protocol
     * @return list of leading protocols without the trailing :
     */
    static public List<String>
    getProtocols(String url, int[] breakpoint)
    {
        // break off any leading protocols;
        // there may be more than one.
        // Watch out for Windows paths starting with a drive letter.
        // Each protocol has trailing ':'  removed

        List<String> allprotocols = new ArrayList<>(); // all leading protocols upto path or host

        // Note, we cannot use split because of the context sensitivity
        StringBuilder buf = new StringBuilder(url);
        int protosize = 0;
        for(; ; ) {
            int index = buf.indexOf(":");
            if(index < 0) break; // no more protocols
            String protocol = buf.substring(0, index);
            // Check for windows drive letter
            if(index == 1 //=>|protocol| == 1 => windows drive letter
                    && "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
                    .indexOf(buf.charAt(0)) >= 0) break;
            allprotocols.add(protocol);
            buf.delete(0, index + 1); // remove the leading protocol
            protosize += (index + 1);
            if(buf.indexOf("/") == 0)
                break; // anything after this is not a protocol
        }
        breakpoint[0] = protosize;
        return allprotocols;
    }

    static public String
    merge(String[] pieces, String sep)
    {
        if(pieces == null) return "";
        StringBuilder buf = new StringBuilder();
        for(int i = 0; i < pieces.length; i++) {
            if(i > 0) buf.append(sep);
            buf.append(pieces[i]);
        }
        return buf.toString();
    }

    static String
    multiSliceString(List<List<Slice>> slices)
    {
        if(slices == null || slices.size() == 0)
            return "[]";
        StringBuilder buf = new StringBuilder();
        for(int i = 0; i < slices.size(); i++) {
            List<Slice> set = slices.get(i);
            buf.append("[");
            for(int j = 0; j < set.size(); j++) {
                if(i > 0)
                    buf.append(",");
                buf.append(set.get(j));
            }
            buf.append("]");
        }
        buf.append("]");
        return buf.toString();
    }

    /**
     * Re-throw run-time exceptions
     */
    static public void checkruntime(Exception e)
    {
        if(e instanceof RuntimeException)
            throw (RuntimeException) e;
    }

    static public String canonjoin(String prefix, String suffix)
    {
        StringBuilder result = new StringBuilder();
        if(prefix == null) prefix = "";
        if(suffix == null) suffix = "";
        prefix = DapUtil.canonicalpath(prefix);
        suffix = DapUtil.canonicalpath(suffix);
        result.append(prefix);
        if(!prefix.endsWith("/"))
            result.append("/");
        result.append(suffix.startsWith("/") ? suffix.substring(1) : suffix);
        return result.toString();
    }

    static public boolean
    hasWindowsDrive(final String path)
    {
        if(path != null && path.length() >= 2 && path.charAt(1) == ':') {
            final char c = path.charAt(0);
            return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
        }
        return false;
    }

    static public boolean
    isAbsolutePath(final String path)
    {
        return path != null && (hasWindowsDrive(path)
                || path.charAt(0) == '/'
                || path.charAt(0) == '\\');
    }

    static public String canonFileURL(String url)
    {
        if(url == null || url.startsWith("file:"))
            return url;
        return "file:" + absolutize(url);
    }

    //////////////////////////////////////////////////

    /**
     * Helper functions for reading
     */

    /**
     * Provide a helper function to convert an Index object to
     * a slice list.
     *
     * @param indices  indicate value to read
     * @param template variable template
     * @return corresponding List<Slice>
     * @throws DapException
     */

    static public List<Slice>
    indexToSlices(Index indices, DapVariable template)
            throws dap4.core.util.DapException
    {
        List<DapDimension> dims = template.getDimensions();
        List<Slice> slices = indexToSlices(indices, dims);
        return slices;
    }

    static public List<Slice>
    indexToSlices(Index indices, List<DapDimension> dimset)
            throws dap4.core.util.DapException
    {
        List<Slice> slices = indexToSlices(indices);
        return slices;
    }

    /**
     * Provide a helper function to convert an offset  to
     * a slice list.
     *
     * @param offset
     * @param template variable template
     * @return
     * @throws dap4.core.util.DapException
     */
    static public List<Slice>
    offsetToSlices(long offset, DapVariable template)
            throws DapException
    {
        List<DapDimension> dims = template.getDimensions();
        long[] dimsizes = DapUtil.getDimSizes(dims);
        return indexToSlices(offsetToIndex(offset, dimsizes), template);
    }

    /**
     * Given an offset (single index) and a set of dimensions
     * compute the set of dimension indices that correspond
     * to the offset.
     */

    static public Index
    offsetToIndex(long offset, long[] dimsizes)
    {
        // offset = d3*(d2*(d1*(x1))+x2)+x3
        long[] indices = new long[dimsizes.length];
        for(int i = dimsizes.length - 1; i >= 0; i--) {
            indices[i] = offset % dimsizes[i];
            offset = (offset - indices[i]) / dimsizes[i];
        }
        return new Index(indices, dimsizes);
    }

    /**
     * Given an offset (single index) and a set of dimensions
     * compute the set of dimension indices that correspond
     * to the offset.
     */

    static public List<Slice>
    indexToSlices(Index indices)
            throws DapException
    {
        // short circuit the scalar case
        if(indices.getRank() == 0)
            return Slice.SCALARSLICES;
        // offset = d3*(d2*(d1*(x1))+x2)+x3
        List<Slice> slices = new ArrayList<>(indices.rank);
        for(int i = 0; i < indices.rank; i++) {
            long isize = indices.indices[i];
            slices.add(new Slice(isize, isize + 1, 1, indices.dimsizes[i]));
        }
        return slices;
    }

    /**
     * Test if a set of slices represent a contiguous region
     * This is equivalent to saying all strides are one
     *
     * @param slices
     * @return
     */
    static public boolean
    isContiguous(List<Slice> slices)
    {
        for(Slice sl : slices) {
            if(sl.getStride() != 1) return false;
        }
        return true;
    }

    /**
     * Test if a set of slices represent a single position
     *
     * @param slices
     * @return
     */
    static public boolean
    isSinglePoint(List<Slice> slices)
    {
        for(Slice sl : slices) {
            if(sl.getCount() != 1) return false;
        }
        return true;
    }

    /**
     * If a set of slices refers to a single position,
     * then return the corresponding Index. Otherwise,
     * throw Exception.
     *
     * @param slices
     * @return Index corresponding to slices
     * @throws DapException
     */
    static public Index
    slicesToIndex(List<Slice> slices)
            throws DapException
    {
        long[] positions = new long[slices.size()];
        long[] dimsizes = new long[slices.size()];
        for(int i = 0; i < positions.length; i++) {
            Slice s = slices.get(i);
            if(s.getCount() != 1)
                throw new DapException("Attempt to convert non-singleton sliceset to index");
            positions[i] = s.getFirst();
            dimsizes[i] = s.getMax();
        }
        return new Index(positions, dimsizes);
    }

    static public List<Slice>
    dimsetToSlices(List<DapDimension> dimset)
            throws dap4.core.util.DapException
    {
        if(dimset == null || dimset.size() == 0)
            return Slice.SCALARSLICES;
        List<Slice> slices = new ArrayList<Slice>(dimset.size());
        for(int i = 0; i < dimset.size(); i++) {
            DapDimension dim = dimset.get(i);
            Slice s = new Slice(dim);
            slices.add(s);
        }
        return slices;
    }

    static public boolean
    isScalarSlices(List<Slice> slices)
    {
        if(slices == null || slices.size() != 1) return false;
        Slice s = slices.get(0);
        return (s.getFirst() == 0 && s.getStop() == 1);
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFileSubclass;
import ucar.nc2.Structure;
import ucar.ma2.*;

import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.InputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.zip.InflaterInputStream;

import com.google.protobuf.InvalidProtocolBufferException;
import ucar.nc2.constants.CDM;

/**
 * Read an ncStream InputStream into a NetcdfFile.
 * Used by CdmRemote
 *
 * @author caron
 * @since Feb 7, 2009
 */
public class NcStreamReader {
  static private final Logger logger = LoggerFactory.getLogger(NcStreamReader.class);

  private static final boolean debug = false;
  private static final boolean showDeflate = false;

  private static double total_uncompressedSize = 0.0;
  private static double total_compressedSize = 0.0;

  static public double getCompression(boolean reset) {
    double result = total_uncompressedSize / total_compressedSize;
    if (reset) {
      total_compressedSize = 0;
      total_uncompressedSize = 0;
    }
    return result;
  }

  public NetcdfFile readStream(InputStream is, NetcdfFile ncfile) throws IOException {
    byte[] b = new byte[4];
    NcStream.readFully(is, b);

    // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER
    if (NcStream.test(b, NcStream.MAGIC_START)) {
      if (!NcStream.readAndTest(is, NcStream.MAGIC_HEADER))
        throw new IOException("Data corrupted on " + ncfile.getLocation());

    } else {
      if (!NcStream.test(b, NcStream.MAGIC_HEADER))
        throw new IOException("Data corrupted on " + ncfile.getLocation());
    }

    // header
    int msize = NcStream.readVInt(is);
    byte[] m = new byte[msize];
    NcStream.readFully(is, m);

    if (debug) {
      System.out.println("READ header len= " + msize);
    }

    NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);
    ncfile = proto2nc(proto, ncfile);
    if (debug) System.out.printf("  proto= %s%n", proto);

    // LOOK why doesnt this work ?
    //CodedInputStream cis = CodedInputStream.newInstance(is);
    //cis.setSizeLimit(msize);
    //NcStreamProto.Stream proto = NcStreamProto.Stream.parseFrom(cis);

    /* why are we reading then ignoring the data?
    while (is.available() > 0) {
      readData(is, ncfile, ncfile.getLocation());
    } */

    return ncfile;
  }

  static public class DataResult {
    public String varNameFullEsc;
    public Array data;

    DataResult(String varName, Array data) {
      this.varNameFullEsc = varName;
      this.data = data;
    }
  }

  /**
   * Read the result of a data request. Only one variable at a time.
   *
   * @param is     read from input stream
   * @param ncfile need the metadata from here to interpret structure data
   * @return DataResult
   * @throws IOException on read error
   */
  public DataResult readData(InputStream is, NetcdfFile ncfile, String location) throws IOException {
    byte[] b = new byte[4];
    int bytesRead = NcStream.readFully(is, b);
    if (bytesRead < b.length)
      throw new EOFException(location);

    if (NcStream.test(b,NcStream.MAGIC_DATA)) return readData1(is, ncfile);
    if (NcStream.test(b,NcStream.MAGIC_DATA2)) return readData2(is);

    throw new IOException("Data transfer corrupted on " + location);
  }

  private DataResult readData1(InputStream is, NetcdfFile ncfile) throws IOException {
    int psize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data message len= " + psize);
    byte[] dp = new byte[psize];
    NcStream.readFully(is, dp);
    NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
    ByteOrder bo = NcStream.decodeDataByteOrder(dproto); // LOOK not using bo !!

    DataType dataType = NcStream.convertDataType(dproto.getDataType());
    Section section = (dataType == DataType.SEQUENCE) ? new Section() : NcStream.decodeSection(dproto.getSection());

    // special cases
    if (dataType == DataType.STRING) {
      int nobjs = NcStream.readVInt(is);
      Array data = Array.factory(dataType, section.getShape());
      assert nobjs == section.computeSize();
      IndexIterator ii = data.getIndexIterator();
      while (ii.hasNext()) {
        int slen = NcStream.readVInt(is);
        byte[] sb = new byte[slen];
        NcStream.readFully(is, sb);
        ii.setObjectNext(new String(sb, CDM.utf8Charset));
      }
      return new DataResult(dproto.getVarName(), data);

    } else if (dataType == DataType.OPAQUE) {
      int nobjs = NcStream.readVInt(is);
      Array data = Array.factory(dataType, section.getShape());
      assert nobjs == section.computeSize();
      IndexIterator ii = data.getIndexIterator();
      while (ii.hasNext()) {
        int slen = NcStream.readVInt(is);
        byte[] sb = new byte[slen];
        NcStream.readFully(is, sb);
        ii.setObjectNext(ByteBuffer.wrap(sb));
      }
      return new DataResult(dproto.getVarName(), data);
    }

    // otherwise read data message
    int dsize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data len= " + dsize);
    byte[] datab = new byte[dsize];
    NcStream.readFully(is, datab);

    if (dataType == DataType.STRUCTURE) {
      Structure s = (Structure) ncfile.findVariable(dproto.getVarName());
      StructureMembers members = s.makeStructureMembers();

      if (dproto.getVersion() == 0) {
        ArrayStructureBB.setOffsets(members); // not setting heap objects for version 0
        ArrayStructureBB data = new ArrayStructureBB(members, section.getShape(), ByteBuffer.wrap(datab), 0);
        return new DataResult(dproto.getVarName(), data);

      } else { // version > 0 and < 3 uses a NcStreamProto.StructureData message
        ArrayStructureBB data = NcStream.decodeArrayStructure(members, section.getShape(), datab);
        return new DataResult(dproto.getVarName(), data);
      }
    }

    // otherwise its a multidim array

    // is it compressed ?
    Array data;
    NcStreamProto.Compress compress = dproto.getCompress();
    int uncompressedSize = dproto.getUncompressedSize();
    if (compress == NcStreamProto.Compress.DEFLATE) {
      ByteArrayInputStream bin = new ByteArrayInputStream(datab);
      InflaterInputStream in = new InflaterInputStream(bin);
      byte[] resultb = new byte[uncompressedSize];
      NcStream.readFully(in, resultb);

      data = Array.factory(dataType, section.getShape(), ByteBuffer.wrap(resultb)); // another copy, not sure can do anything
      if (showDeflate)
        System.out.printf("Deflate = %d / %d = %f %n", uncompressedSize, dsize, ((float) uncompressedSize) / dsize);
      total_uncompressedSize += uncompressedSize;
      total_compressedSize += dsize;

    } else {
      data = Array.factory(dataType, section.getShape(), ByteBuffer.wrap(datab));
    }

    return new DataResult(dproto.getVarName(), data);
  }

  private DataResult readData2(InputStream is) throws IOException {
    int psize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data message len= " + psize);
    byte[] dp = new byte[psize];
    NcStream.readFully(is, dp);

    NcStreamProto.DataCol dproto = NcStreamProto.DataCol.parseFrom(dp);
    // NcStreamProto.Data2 dproto = NcStreamProto.Data2.parseDelimitedFrom(is);

    NcStreamDataCol decoder = new NcStreamDataCol();
    Array data = decoder.decode(dproto, null);
    return new DataResult(dproto.getName(), data);
  }

  ////////////////////////////////////////////////////////////////////////////////////////
  // LOOK

  public StructureDataIterator getStructureIterator(InputStream is, NetcdfFile ncfile) throws IOException {
    if (!NcStream.readAndTest(is, NcStream.MAGIC_DATA))
      throw new IOException("Data transfer corrupted on " + ncfile.getLocation());

    int psize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data message len= " + psize);
    byte[] dp = new byte[psize];
    NcStream.readFully(is, dp);
    NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
    // if (debug) System.out.println(" readData proto = " + dproto);

    Structure s = (Structure) ncfile.findVariable(dproto.getVarName());
    StructureMembers members = s.makeStructureMembers();
    ArrayStructureBB.setOffsets(members);

    ByteOrder bo = NcStream.decodeDataByteOrder(dproto);
    return new StreamDataIterator(is, members, bo);
  }

  private static class StreamDataIterator implements StructureDataIterator {
    private InputStream is;
    private StructureMembers members;
    private StructureData curr = null;
    private ByteOrder bo;
    private int count = 0;
    private boolean done;

    StreamDataIterator(InputStream is, StructureMembers members, ByteOrder bo) {
      this.is = is;
      this.members = members;
      this.bo = bo;
    }

    @Override
    public boolean hasNext() throws IOException {
      if (!done) readNext();
      return (curr != null);
    }

    @Override
    public StructureData next() throws IOException {
      count++;
      return curr;
    }

    private void readNext() throws IOException {
      byte[] b = new byte[4];
      NcStream.readFully(is, b);

      // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER
      if (NcStream.test(b, NcStream.MAGIC_VDATA)) {
        int dsize = NcStream.readVInt(is);
        byte[] datab = new byte[dsize];
        NcStream.readFully(is, datab);
        // curr = NcStream.decodeStructureData(members, bo, datab); LOOK
        // System.out.printf("StreamDataIterator read sdata size= %d%n", dsize);

      } else if (NcStream.test(b, NcStream.MAGIC_VEND)) {
        curr = null;
        close();

      } else {
        throw new IllegalStateException("bad stream");
      }
    }

    @Override
    public StructureDataIterator reset() {
      return (count == 0) && (is != null) ? this : null;
    }

    @Override
    public int getCurrentRecno() {
      return count;
    }

    @Override
    public void close() {
      done = true;
      if (is != null) {
        try {
          is.close();
          is = null;
        } catch (IOException ioe) {
          logger.error("NcStreamReader: Error closing input stream.");
        }
      }
    }
  }

  /////////////////////////////////////////////////////////////////////

  private NetcdfFile proto2nc(NcStreamProto.Header proto, NetcdfFile ncfile) throws InvalidProtocolBufferException {
    if (ncfile == null)
      ncfile = new NetcdfFileSubclass(); // not used i think
    ncfile.setLocation(proto.getLocation());
    if (proto.getId().length() > 0) ncfile.setId(proto.getId());
    if (proto.getTitle().length() > 0) ncfile.setTitle(proto.getTitle());

    NcStreamProto.Group root = proto.getRoot();
    NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
    ncfile.finish();
    return ncfile;
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.point.standard;

import org.jdom2.output.XMLOutputter;
import org.jdom2.output.Format;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;

import java.io.IOException;
import java.io.InputStream;
import java.io.FileNotFoundException;
import java.util.List;
import java.util.ArrayList;
import java.util.Formatter;

import ucar.nc2.ft.FeatureDatasetPoint;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.VariableDS;
import ucar.nc2.util.IO;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.Dimension;
import ucar.nc2.Variable;
import ucar.nc2.Structure;
import ucar.ma2.DataType;

/**
 * Helper class to convert a TableConfig to and from XML
 *
 * @author caron
 * @since Aug 18, 2009
 */
public class PointConfigXML {

  public static void writeConfigXML(FeatureDatasetPoint pfd, java.util.Formatter f) {
    if (!(pfd instanceof PointDatasetStandardFactory.PointDatasetStandard)) {
      f.format("%s not instance of PointDatasetStandard%n", pfd.getLocation());
      return;
    }
    PointDatasetStandardFactory.PointDatasetStandard spfd = (PointDatasetStandardFactory.PointDatasetStandard) pfd;
    TableAnalyzer analyser = spfd.getTableAnalyzer();
    TableConfig config = analyser.getTableConfig();
    TableConfigurer tc = analyser.getTableConfigurer();
    if (tc == null) {
      f.format("%s has no TableConfig%n", pfd.getLocation());
      return;
    }

    PointConfigXML writer = new PointConfigXML();
    try {
      writer.writeConfigXML(config, tc.getClass().getName(), f);
    } catch (IOException e) {
      f.format("%s error writing=%s%n", pfd.getLocation(), e.getMessage());
    }
  }

  ////////////////////////////////////////////////////////////////////
  private TableConfig tc;
  private String tableConfigurerClass;

  public void writeConfigXML(TableConfig tc, String tableConfigurerClass, java.util.Formatter sf) throws IOException {
    this.tc = tc;
    this.tableConfigurerClass = tableConfigurerClass;

    XMLOutputter fmt = new XMLOutputter(Format.getPrettyFormat());
    sf.format("%s", fmt.outputString(makeDocument()));
  }

  /**
   * Create an XML document from this info
   *
   * @return netcdfDatasetInfo XML document
   */
  public Document makeDocument() {
    Element rootElem = new Element("pointConfig");
    Document doc = new Document(rootElem);
    if (tableConfigurerClass != null)
      rootElem.addContent( new Element("tableConfigurer").setAttribute("class", tableConfigurerClass));
    if (tc.featureType != null)
      rootElem.setAttribute("featureType", tc.featureType.toString());

    rootElem.addContent(writeTable(tc));

    return doc;
  }

  private Element writeTable(TableConfig config) {
    Element tableElem = new Element("table");

    //if (config.name != null)
    //  tableElem.setAttribute("name", config.name);
    if (config.type != null) {
      tableElem.setAttribute("type", config.type.toString());

      switch (config.type) {
        case ArrayStructure:
          tableElem.setAttribute("dimension", config.dimName);
          break;

        case Construct:
          break;

        case Contiguous:
          if (config.start != null)
            tableElem.setAttribute("start", config.start);

          if (config.numRecords != null)
            tableElem.setAttribute("numRecords", config.numRecords);
          break;

        case LinkedList:
          tableElem.setAttribute("start", config.start);
          tableElem.setAttribute("next", config.next);
          break;

        case MultidimInner:
          tableElem.setAttribute("structName", config.structName);
          //coverity[missing_break]

        case MultidimInnerPsuedo:
          tableElem.setAttribute("dim0", config.outerName);
          tableElem.setAttribute("dim1", config.innerName);
          tableElem.setAttribute("subtype", config.structureType.toString());
          break;

        case MultidimInner3D:
          break;

        case MultidimInnerPsuedo3D:
          break;

        case MultidimStructure:
          tableElem.setAttribute("structName", config.structName);
          break;

        case NestedStructure:
          tableElem.setAttribute("structName", config.structName);
          break;

        case ParentId:
        case ParentIndex:
          tableElem.setAttribute("parentIndex", config.parentIndex);
          break;

        case Singleton:
          break;

        case Structure:
          tableElem.setAttribute("subtype", config.structureType.toString());
          switch (config.structureType) {
            case Structure:
              tableElem.setAttribute("structName", config.structName);
              break;
            case PsuedoStructure:
              tableElem.setAttribute("dim", config.dimName);
              break;
            case PsuedoStructure2D:
              tableElem.setAttribute("dim0", config.dimName);
              tableElem.setAttribute("dim1", config.outerName);
              break;
          }
          break;

        case Top:
          tableElem.setAttribute("structName", config.structName);
          break;

      }
    }

    List<String> varNames = (config.vars == null) ? new ArrayList<String>() : new ArrayList<>(config.vars);

    // add coordinates
    for (Table.CoordName coord : Table.CoordName.values()) {
      addCoord(tableElem, config, coord, varNames);
    }

    // add variables
    if (config.vars != null) {
      for (String col : varNames)
        tableElem.addContent(new Element("variable").addContent(col));
    }

    // add joins
    if (config.extraJoin != null) {
      for (Join j : config.extraJoin) {
        if (j instanceof JoinArray)
          tableElem.addContent(writeJoinArray((JoinArray) j));
        else if (j instanceof JoinMuiltdimStructure)
          tableElem.addContent(writeJoinMuiltdimStructure((JoinMuiltdimStructure) j));
        else if (j instanceof JoinParentIndex)
          tableElem.addContent(writeJoinParentIndex((JoinParentIndex) j));
      }
    }

    if (config.children != null) {
      for (TableConfig child : config.children)
        tableElem.addContent(writeTable(child));
    }
    return tableElem;
  }

  private void addCoord(Element tableElem, TableConfig table, Table.CoordName type, List<String> varNames) {
    String name = table.findCoordinateVariableName(type);
    if (name != null) {
      Element elem = new Element("coordinate").setAttribute("type", type.name());
      elem.addContent(name);
      tableElem.addContent(elem);
      varNames.remove(name);
    }
  }

  private Element writeJoinArray(JoinArray join) {
    Element joinElem = new Element("join");
    joinElem.setAttribute("class", join.getClass().toString());
    if (join.type != null)
      joinElem.setAttribute("type", join.type.toString());
    if (join.v != null)
      joinElem.addContent(new Element("variable").addContent(join.v.getFullName()));
    joinElem.addContent(new Element("param").addContent(Integer.toString(join.param)));
    return joinElem;
  }

  private JoinArray readJoinArray(NetcdfDataset ds, Element joinElement) {
    JoinArray.Type type = JoinArray.Type.valueOf(joinElement.getAttributeValue("type"));
    Element paramElem = joinElement.getChild("param");
    String paramS = paramElem.getText();
    Integer param = Integer.parseInt(paramS);

    Element varElem = joinElement.getChild("variable");
    String varName = varElem.getText();
    VariableDS v = (VariableDS) ds.findVariable(varName);
    return new JoinArray(v, type, param);
  }

  private Element writeJoinMuiltdimStructure(JoinMuiltdimStructure join) {
    Element joinElem = new Element("join");
    joinElem.setAttribute("class", join.getClass().toString());
    if (join.parentStructure != null)
      joinElem.addContent(new Element("parentStructure").addContent(join.parentStructure.getFullName()));
    joinElem.addContent(new Element("dimLength").setAttribute("value", Integer.toString(join.dimLength)));
    return joinElem;
  }

  private Element writeJoinParentIndex(JoinParentIndex join) {
    Element joinElem = new Element("join");
    joinElem.setAttribute("class", join.getClass().toString());
    if (join.parentStructure != null)
      joinElem.addContent(new Element("parentStructure").addContent(join.parentStructure.getFullName()));
    if (join.parentIndex != null)
      joinElem.addContent(new Element("parentIndex").addContent(join.parentIndex));
    return joinElem;
  }

  ///////////////////////////////////////////////////////////////////////////
  private static boolean debugXML = false;
  private static boolean debugURL = false;
  private static boolean showParsedXML = false;

  public TableConfig readConfigXMLfromResource(String resourceLocation, FeatureType wantFeatureType, NetcdfDataset ds, Formatter errlog) throws IOException {
    ClassLoader cl = this.getClass().getClassLoader();
    try (InputStream is = cl.getResourceAsStream(resourceLocation)) {
      if (is == null)
        throw new FileNotFoundException(resourceLocation);

      if (debugXML) {
        System.out.println(" PointConfig URL = <" + resourceLocation + ">");
        try (InputStream is2 = cl.getResourceAsStream(resourceLocation)) {
          System.out.println(" contents=\n" + IO.readContents(is2));
        }
      }

      org.jdom2.Document doc;
      try {
        SAXBuilder builder = new SAXBuilder();
        if (debugURL) System.out.println(" PointConfig URL = <" + resourceLocation + ">");
        doc = builder.build(is);
      } catch (JDOMException e) {
        throw new IOException(e.getMessage());
      }
      if (debugXML) System.out.println(" SAXBuilder done");

      if (showParsedXML) {
        XMLOutputter xmlOut = new XMLOutputter();
        System.out.println("*** PointConfig/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
      }

      Element configElem = doc.getRootElement();
      String featureType = configElem.getAttributeValue("featureType");
      Element tableElem = configElem.getChild("table");
      assert tableElem != null;
      TableConfig tc = parseTableConfig(ds, tableElem, null);
      tc.featureType = FeatureType.valueOf(featureType);
      return tc;
    }

  }

  public TableConfig readConfigXML(String fileLocation, FeatureType wantFeatureType, NetcdfDataset ds, Formatter errlog) throws IOException {

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      if (debugURL) System.out.println(" PointConfig URL = <" + fileLocation + ">");
      doc = builder.build(fileLocation);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML) System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** PointConfig/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element configElem = doc.getRootElement();
    String featureType = configElem.getAttributeValue("featureType");
    Element tableElem = configElem.getChild("table");
    TableConfig tc = parseTableConfig(ds, tableElem, null);
    tc.featureType = FeatureType.valueOf(featureType);

    return tc;
  }

  private TableConfig parseTableConfig(NetcdfDataset ds, Element tableElem, TableConfig parent) {
    String typeS = tableElem.getAttributeValue("type");
    Table.Type ttype = Table.Type.valueOf(typeS);

    String name = tableElem.getAttributeValue("name");
    TableConfig tc = new TableConfig(ttype, name);

    switch (ttype) {
      case ArrayStructure:
        tc.dimName = tableElem.getAttributeValue("dimension");
        break;

      case Construct:
        break;

      case Contiguous:
        tc.numRecords = tableElem.getAttributeValue("numRecords");
        tc.start = tableElem.getAttributeValue("start");
        break;

      case LinkedList:
        tc.start = tableElem.getAttributeValue("start");
        tc.next = tableElem.getAttributeValue("next");
        break;

      case MultidimInner:
        tc.structName = tableElem.getAttributeValue("structName");
        //coverity[missing_break]

      case MultidimInnerPsuedo:
        tc.outerName = tableElem.getAttributeValue("dim0");
        tc.dimName = tc.outerName;
        tc.innerName = tableElem.getAttributeValue("dim1");
        tc.structureType = TableConfig.StructureType.valueOf(tableElem.getAttributeValue("subtype"));
        assert parent != null;
        makeMultidimInner(ds, parent, tc);
        break;

      case MultidimInner3D:
        break;

      case MultidimInnerPsuedo3D:
        break;

      case MultidimStructure:
        tc.structName = tableElem.getAttributeValue("structName");
        break;

      case NestedStructure:
        tc.structName = tableElem.getAttributeValue("structName");
        break;

      case ParentId:
      case ParentIndex:
        tc.parentIndex = tableElem.getAttributeValue("parentIndex");
        break;

      case Singleton:
        break;

      case Structure:
        tc.structureType = TableConfig.StructureType.valueOf(tableElem.getAttributeValue("subtype"));
        switch (tc.structureType) {
          case Structure:
            tc.structName = tableElem.getAttributeValue("structName");
            break;
          case PsuedoStructure:
            tc.dimName = tableElem.getAttributeValue("dim");
            break;
          case PsuedoStructure2D:
            tc.dimName = tableElem.getAttributeValue("dim0");
            tc.outerName = tableElem.getAttributeValue("dim1");
            break;
        }
        break;

      case Top:
        tc.structName = tableElem.getAttributeValue("structName");
        break;

    }

    List<Element> coordList = tableElem.getChildren("coordinate");
    for (Element coordElem : coordList) {
      String coordNameType = coordElem.getAttributeValue("type");
      Table.CoordName coordName = Table.CoordName.valueOf(coordNameType);
      tc.setCoordinateVariableName(coordName, coordElem.getText());
    }

    List<Element> joinList = tableElem.getChildren("join");
    for (Element joinElem : joinList) {
      tc.addJoin( readJoinArray(ds, joinElem));
    }

    List<Element> nestedTableList = tableElem.getChildren("table");
    for (Element nestedTable : nestedTableList)
      tc.addChild( parseTableConfig(ds, nestedTable, tc));

    return tc;
  }

  private void makeMultidimInner(NetcdfDataset ds, TableConfig parentTable, TableConfig childTable) {
    Dimension parentDim = ds.findDimension(parentTable.dimName);
    Dimension childDim = ds.findDimension(childTable.innerName);

    // divide up the variables between the parent and the child
    List<String> obsVars;
    List<Variable> vars = ds.getVariables();
    List<String> parentVars = new ArrayList<>(vars.size());
    obsVars = new ArrayList<>(vars.size());
    for (Variable orgV : vars) {
      if (orgV instanceof Structure) continue;

      Dimension dim0 = orgV.getDimension(0);
      if ((dim0 != null) && dim0.equals(parentDim)) {
        if ((orgV.getRank() == 1) || ((orgV.getRank() == 2) && orgV.getDataType() == DataType.CHAR)) {
          parentVars.add(orgV.getShortName());
        } else {
          Dimension dim1 = orgV.getDimension(1);
          if ((dim1 != null) && dim1.equals(childDim))
            obsVars.add(orgV.getShortName());
        }
      }
    }
    parentTable.vars = parentVars;
    childTable.vars = obsVars;
  }

  /* the inner table of Structure(outer, middle, inner)
  private TableConfig makeMultidimInner3D(NetcdfDataset ds, TableConfig outerTable, TableConfig middleTable, Dimension innerDim, Formatter errlog) throws IOException {
    Dimension outerDim = ds.findDimension(outerTable.dimName);
    Dimension middleDim = ds.findDimension(middleTable.innerName);

    Table.Type obsTableType = (outerTable.structureType == TableConfig.StructureType.PsuedoStructure) ? Table.Type.MultidimInnerPsuedo3D : Table.Type.MultidimInner3D;
    TableConfig obsTable = new TableConfig(obsTableType, innerDim.getName());
    obsTable.structureType = TableConfig.StructureType.PsuedoStructure2D;
    obsTable.dimName = outerTable.dimName;
    obsTable.outerName = middleTable.innerName;
    obsTable.innerName = innerDim.getName();
    obsTable.structName = innerDim.getName();

    obsTable.lat = matchAxisTypeAndDimension(ds, AxisType.Lat, outerDim, middleDim, innerDim);
    obsTable.lon = matchAxisTypeAndDimension(ds, AxisType.Lon, outerDim, middleDim, innerDim);
    obsTable.elev = matchAxisTypeAndDimension(ds, AxisType.Height, outerDim, middleDim, innerDim);
    obsTable.time = matchAxisTypeAndDimension(ds, AxisType.Time, outerDim, middleDim, innerDim);

    // divide up the variables between the 3 tables
    List<Variable> vars = ds.getVariables();
    List<String> outerVars = new ArrayList<String>(vars.size());
    List<String> middleVars = new ArrayList<String>(vars.size());
    List<String> innerVars = new ArrayList<String>(vars.size());
    for (Variable orgV : vars) {
      if (orgV instanceof Structure) continue;

      if ((orgV.getRank() == 1) || ((orgV.getRank() == 2) && orgV.getDataType() == DataType.CHAR)) {
        if (outerDim.equals(orgV.getDimension(0)))
          outerVars.add(orgV.getShortName());

      } else if (orgV.getRank() == 2) {
        if (outerDim.equals(orgV.getDimension(0)) && middleDim.equals(orgV.getDimension(1)))
          middleVars.add(orgV.getShortName());

      } else if (orgV.getRank() == 3) {
        if (outerDim.equals(orgV.getDimension(0)) && middleDim.equals(orgV.getDimension(1)) && innerDim.equals(orgV.getDimension(2)))
          innerVars.add(orgV.getShortName());
      }
    }
    outerTable.vars = outerVars;
    middleTable.vars = middleVars;
    obsTable.vars = innerVars;

    return obsTable;
  } */

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.point.bufr;

import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.bufr.BufrConfig;
import ucar.nc2.stream.NcStream;
import ucar.nc2.time.CalendarDate;
import ucar.unidata.io.RandomAccessFile;

import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Manage cdm index (ncx) for Bufr files.
 * Covers BufrCdmIndexProto
 * Never completes or in operational use, could redo as needed
 *
 * @author caron
 * @since 8/14/13
 */
public class BufrCdmIndex {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(BufrCdmIndex.class);

  public static final String MAGIC_START = "BufrCdmIndex";
  public static final String NCX_IDX = ".ncx";
  private static final int version = 1;

  public static File calcIndexFile(String bufrFilename ) {
    File bufrFile = new File(bufrFilename);
    String name = bufrFile.getName();
    File result =  new File(bufrFile.getParent(), name + BufrCdmIndex.NCX_IDX);
    if (result.exists()) return result;

    int pos = name.indexOf('.');
    if (pos > 0) {
      name = name.substring(0, pos);
      result =  new File(bufrFile.getParent(), name + BufrCdmIndex.NCX_IDX);
      if (result.exists()) return result;
    }

    return null;
  }

  public static boolean writeIndex(String bufrFilename, BufrConfig config, File idxFile) throws IOException {
    return new BufrCdmIndex().writeIndex2(bufrFilename, config, idxFile);
  }

  public static BufrCdmIndex readIndex(String indexFilename) throws IOException {
    BufrCdmIndex index =  new BufrCdmIndex();
    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFilename)) {
       index.readIndex(raf);
    }
    return index;
  }

  /////////////////////////////////////////////////////////////////////////////////

  /*
   MAGIC_START
   version
   sizeIndex
   BufrCdmIndexProto (sizeIndex bytes)
  */
  private boolean writeIndex2(String bufrFilename, BufrConfig config, File indexFile) throws IOException {
    if (indexFile.exists()) {
      if (!indexFile.delete())
        log.warn(" BufrCdmIndex cant delete index file {}", indexFile.getPath());
    }
    log.debug(" createIndex for {}", indexFile.getPath());

    try (RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw")) {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      //// header message
      raf.write(MAGIC_START.getBytes(CDM.utf8Charset));
      raf.writeInt(version);

      // build it
      BufrCdmIndexProto.BufrIndex.Builder indexBuilder = BufrCdmIndexProto.BufrIndex.newBuilder();
      indexBuilder.setFilename(bufrFilename);
      root = buildField(config.getRootConverter());
      indexBuilder.setRoot(root);
      indexBuilder.setStart(config.getStart());
      indexBuilder.setEnd(config.getEnd());
      indexBuilder.setNobs(config.getNobs());

      Map<String,BufrConfig.BufrStation> smaps = config.getStationMap();
      if (smaps != null) {
        List<BufrConfig.BufrStation> stations = new ArrayList<BufrConfig.BufrStation>(smaps.values());
        Collections.sort(stations);
        for (BufrConfig.BufrStation s : stations) {
          indexBuilder.addStations(buildStation(s));
        }
      }

      // write it
      BufrCdmIndexProto.BufrIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp

      log.debug("  file size =  %d bytes", raf.length());
      return true;
    }
  }

  public static boolean writeIndex(BufrCdmIndex index, BufrField root, File indexFile) throws IOException {
    if (indexFile.exists()) {
      if (!indexFile.delete())
        log.warn(" BufrCdmIndex cant delete index file {}", indexFile.getPath());
    }
    log.debug(" createIndex for {}", indexFile.getPath());

    try (RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw")) {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      //// header message
      raf.write(MAGIC_START.getBytes(CDM.utf8Charset));
      raf.writeInt(version);

      // build it
      BufrCdmIndexProto.BufrIndex.Builder indexBuilder = BufrCdmIndexProto.BufrIndex.newBuilder();
      indexBuilder.setFilename(index.bufrFilename);
      BufrCdmIndexProto.Field rootf = buildField(root);
      indexBuilder.setRoot( rootf);
      indexBuilder.setStart(index.start);
      indexBuilder.setEnd(index.end);
      indexBuilder.setNobs(index.nobs);

      if (index.stations != null) {
        for (BufrCdmIndexProto.Station s : index.stations) {
          indexBuilder.addStations(s);
        }
      }

      // write it
      BufrCdmIndexProto.BufrIndex indexOut = indexBuilder.build();
      byte[] b = indexOut.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp
      log.debug("  write BufrCdmIndexProto= {} bytes", b.length);

      //System.out.printf("  write BufrCdmIndexProto= %d bytes", b.length);
      //showProtoRoot(rootf);

      log.debug("  file size =  %d bytes", raf.length());
      return true;
    }
  }

  private BufrCdmIndexProto.Station buildStation(BufrConfig.BufrStation s) {
    BufrCdmIndexProto.Station.Builder builder = BufrCdmIndexProto.Station.newBuilder();

    builder.setId(s.getName());
    builder.setCount(s.count);
    if (s.getWmoId() != null)
      builder.setWmoId(s.getWmoId());
    if (s.getDescription() != null)
      builder.setDesc(s.getDescription());
    builder.setLat( s.getLatitude());
    builder.setLon(s.getLongitude());
    builder.setAlt(s.getAltitude());

    return builder.build();
  }

  private static BufrCdmIndexProto.Field buildField(BufrField fld) {
    BufrCdmIndexProto.Field.Builder fldBuilder = BufrCdmIndexProto.Field.newBuilder();

    fldBuilder.setFxy(fld.getFxy());
    fldBuilder.setScale(fld.getScale());
    fldBuilder.setReference(fld.getReference());
    fldBuilder.setBitWidth(fld.getBitWidth());

    if (fld.getName() != null)
      fldBuilder.setName(fld.getName());

    if (fld.getDesc() != null)
      fldBuilder.setDesc(fld.getDesc());

    if (fld.getUnits() != null)
      fldBuilder.setUnits(fld.getUnits());

    if (fld.getChildren() != null) {
      for (BufrField child : fld.getChildren())
        fldBuilder.addFlds(buildField(child));
    }

    if (fld.getAction() != null && fld.getAction() != BufrCdmIndexProto.FldAction.none)
      fldBuilder.setAction( fld.getAction());

    if (fld.getType() != null)
      fldBuilder.setType(fld.getType());

    if (fld.isSeq()) {
      fldBuilder.setMin( fld.getMin());
      fldBuilder.setMax(fld.getMax());
    }

    return fldBuilder.build();
  }

  //////////////////////////////////////////////////////////////////

  public String idxFilename;
  public String bufrFilename;
  public BufrCdmIndexProto.Field root;
  public List<BufrCdmIndexProto.Station> stations;
  public long start, end;
  public long nobs;

  protected boolean readIndex(RandomAccessFile raf) throws IOException {
    this.idxFilename = raf.getLocation();

    try {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      //// header message
      if (!NcStream.readAndTest(raf, MAGIC_START.getBytes(CDM.utf8Charset))) {
        log.error("BufrCdmIndex {}: invalid index", raf.getLocation());
        return false;
      }

      int indexVersion = raf.readInt();
      boolean versionOk = (indexVersion == version);
      if (!versionOk) {
        log.warn("BufrCdmIndex {}: index found version={}, want version= {}", raf.getLocation(), indexVersion, version);
        return false;
      }

      int size = NcStream.readVInt(raf);
      if ((size < 0) || (size > 100 * 1000 * 1000)) {
        log.warn("BufrCdmIndex {}: invalid or empty index ", raf.getLocation());
        return false;
      }

      byte[] m = new byte[size];
      raf.readFully(m);

      BufrCdmIndexProto.BufrIndex proto =  BufrCdmIndexProto.BufrIndex.parseFrom(m);
      bufrFilename = proto.getFilename();
      root = proto.getRoot();
      stations = proto.getStationsList();
      start = proto.getStart();
      end = proto.getEnd();
      nobs = proto.getNobs();

      //showProtoRoot(root);

    } catch (Throwable t) {
      log.error("Error reading index " + raf.getLocation(), t);
      return false;
    }

    return true;
  }

  static void showProtoRoot(BufrCdmIndexProto.Field fld) {
    String act = (fld.getAction() != BufrCdmIndexProto.FldAction.defa) ? fld.getAction().toString() : "-";
    System.out.printf("%10s %s%n", act, fld.getName());
    for (BufrCdmIndexProto.Field child : fld.getFldsList())
      showProtoRoot(child);

  }

  public void showIndex(Formatter f) {
    f.format("BufrCdmIndex %n");
    f.format("  idxFilename=%s%n", idxFilename);
    f.format("  bufrFilename=%s%n", bufrFilename);
    f.format("  dates=[%s,%s]%n", CalendarDate.of(start), CalendarDate.of(end));
    f.format("  nobs=%s%n", nobs);
    if (stations != null)  {
      f.format("  # stations=%d%n", stations.size());
      int count = 0;
      for (BufrCdmIndexProto.Station s : stations)
        count += s.getCount();
      f.format("  # stations obs=%d%n", count);
    }

  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.ma2;

import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;

/**
 * Concrete implementation of Array specialized for doubles.
 * Data storage is with 1D java array of doubles.
 *
 * @see Array
 * @author caron
 */
public class ArrayDouble extends Array {

  /** package private. use Array.factory() */
  static ArrayDouble factory(Index index) {
    return ArrayDouble.factory(index, null);
  }

  /* Create new ArrayDouble with given indexImpl and backing store.
   * Should be private.
   * @param index use this Index
   * @param stor. use this storage. if null, allocate.
   * @return. new ArrayDouble.D<rank> or ArrayDouble object.
   */
  static ArrayDouble factory( Index index, double [] storage) {
      if (index instanceof Index0D) {
          return new ArrayDouble.D0(index, storage);
      } else if (index instanceof Index1D) {
          return new ArrayDouble.D1(index, storage);
      } else if (index instanceof Index2D) {
          return new ArrayDouble.D2(index, storage);
      } else if (index instanceof Index3D) {
          return new ArrayDouble.D3(index, storage);
      } else if (index instanceof Index4D) {
          return new ArrayDouble.D4(index, storage);
      } else if (index instanceof Index5D) {
          return new ArrayDouble.D5(index, storage);
      } else if (index instanceof Index6D) {
          return new ArrayDouble.D6(index, storage);
      } else if (index instanceof Index7D) {
          return new ArrayDouble.D7(index, storage);
      } else {
          return new ArrayDouble(index, storage);
      }
  }

  ///////////////////////////////////////////////////////////////////////////////
  protected double[] storageD;

  /**
  * Create a new Array of type double and the given shape.
  * dimensions.length determines the rank of the new Array.
  * @param shape the shape of the Array.
  */
  public ArrayDouble(int [] shape) {
    super(DataType.DOUBLE, shape);
    storageD = new double[(int) indexCalc.getSize()];
  }

  /** create new Array with given indexImpl and the same backing store */
  protected Array createView( Index index) {
    return ArrayDouble.factory(index, storageD);
  }

 /**
  * Create a new Array using the given IndexArray and backing store.
  * used for sections, and factory. Trusted package private.
  * @param ima use this IndexArray as the index
  * @param data use this as the backing store. if null, allocate
  */
  ArrayDouble(Index ima, double [] data) {
    super(DataType.DOUBLE, ima);

    if (data != null) {
      storageD = data;
    } else {
      storageD = new double[(int) indexCalc.getSize()];
    }
  }

  /* Get underlying primitive array storage. CAUTION! You may invalidate your warrentee! */
  public Object getStorage() { return storageD; }

  // copy from javaArray to storage using the iterator: used by factory( Object);
  protected void copyFrom1DJavaArray(IndexIterator iter, Object javaArray) {
    double[] ja = (double []) javaArray;
    for (double aJa : ja) iter.setDoubleNext(aJa);
  }

  // copy to javaArray from storage using the iterator: used by copyToNDJavaArray;
  protected void copyTo1DJavaArray(IndexIterator iter, Object javaArray) {
    double[] ja = (double []) javaArray;
    for (int i=0; i<ja.length; i++)
      ja[i] = iter.getDoubleNext();
  }

  public ByteBuffer getDataAsByteBuffer() {
    ByteBuffer bb = ByteBuffer.allocate((int)(8*getSize()));
    DoubleBuffer ib = bb.asDoubleBuffer();
    ib.put( (double[]) get1DJavaArray(DataType.DOUBLE)); // make sure its in canonical order
    return bb;
  }

 /** Return the element class type */
  public Class getElementType() { return double.class; }

    /** get the value at the specified index. */
  public double get(Index i) {
    return storageD[i.currentElement()];
  }
    /** set the value at the specified index. */
  public void set(Index i, double value) {
    storageD[i.currentElement()] = value;
  }

  public double getDouble(Index i) {
    return storageD[i.currentElement()];
  }
  public void setDouble(Index i, double value) {
    storageD[i.currentElement()] = value;
  }

  public float getFloat(Index i) {
    return (float) storageD[i.currentElement()];
  }
  public void setFloat(Index i, float value) {
    storageD[i.currentElement()] = (double) value;
  }

  public long getLong(Index i) {
    return (long) storageD[i.currentElement()];
  }
  public void setLong(Index i, long value) {
    storageD[i.currentElement()] = (double) value;
  }

  public int getInt(Index i) {
    return (int) storageD[i.currentElement()];
  }
  public void setInt(Index i, int value) {
    storageD[i.currentElement()] = (double) value;
  }

  public short getShort(Index i) {
    return (short) storageD[i.currentElement()];
  }
  public void setShort(Index i, short value) {
    storageD[i.currentElement()] = (double) value;
  }

  public byte getByte(Index i) {
    return (byte) storageD[i.currentElement()];
  }
  public void setByte(Index i, byte value) {
    storageD[i.currentElement()] = (double) value;
  }

  public char getChar(Index i) {
    return (char) storageD[i.currentElement()];
  }
  public void setChar(Index i, char value) {
    storageD[i.currentElement()] = (double) value;
  }

  /** not legal, throw ForbiddenConversionException */
  public boolean getBoolean(Index i) {
    throw new ForbiddenConversionException();
  }
  /** not legal, throw ForbiddenConversionException */
  public void setBoolean(Index i, boolean value) {
     throw new ForbiddenConversionException();
  }

  public Object getObject(Index i) {
    return storageD[i.currentElement()];
  }
  public void setObject(Index i, Object value) {
    storageD[i.currentElement()] = ((Number)value).doubleValue();
  }

    // trusted, assumes that individual dimension lengths have been checked
  public double getDouble(int index) {return storageD[index]; }
  public void setDouble(int index, double value) { storageD[index] = value; }

  public float getFloat(int index) { return (float) storageD[index]; }
  public void setFloat(int index, float value) { storageD[index] = (double) value;}

  public long getLong(int index) {return (long) storageD[index];}
  public void setLong(int index, long value) { storageD[index] = (double) value;}

  public int getInt(int index) { return (int) storageD[index]; }
  public void setInt(int index, int value) { storageD[index] = (double) value;}

  public short getShort(int index) { return (short) storageD[index]; }
  public void setShort(int index, short value) { storageD[index] = (double) value; }

  public byte getByte(int index) { return (byte) storageD[index]; }
  public void setByte(int index, byte value) {storageD[index] = (double) value;}

  public char getChar(int index) { return (char) storageD[index];}
  public void setChar(int index, char value) { storageD[index] = (double) value; }

  public boolean getBoolean(int index) { throw new ForbiddenConversionException(); }
  public void setBoolean(int index, boolean value) {throw new ForbiddenConversionException(); }

  public Object getObject(int index) { return getDouble(index); }
  public void setObject(int index, Object value) { storageD[index] = ((Number)value).doubleValue(); }

  /** Concrete implementation of Array specialized for doubles, rank 0. */
  public static class D0 extends ArrayDouble {
    private Index0D ix;
    /** Constructor. */
    public D0 () {
      super(new int [] {});
      ix = (Index0D) indexCalc;
    }
    private D0 (Index i, double[] store) {
      super(i, store);
      ix = (Index0D) indexCalc;
    }
    /** get the value. */
    public double get() {
      return storageD[ix.currentElement()];
    }
    /** set the value. */
    public void set(double value) {
      storageD[ix.currentElement()] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 1. */
  public static class D1 extends ArrayDouble {
    private Index1D ix;
    /** Constructor for array of shape {len0}. */
    public D1 (int len0) {
      super(new int [] {len0});
      ix = (Index1D) indexCalc;
    }
    private D1 (Index i, double[] store) {
      super(i, store);
      ix = (Index1D) indexCalc;
    }
    /** get the value. */
    public double get(int i) {
      return storageD[ix.setDirect(i)];
    }
    /** set the value. */
    public void set(int i, double value) {
      storageD[ix.setDirect(i)] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 2. */
  public static class D2 extends ArrayDouble {
    private Index2D ix;
    /** Constructor for array of shape {len0,len1}. */
    public D2 (int len0, int len1) {
      super(new int [] {len0, len1});
      ix = (Index2D) indexCalc;
    }
    private D2 (Index i, double[] store) {
      super(i, store);
      ix = (Index2D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j) {
      return storageD[ix.setDirect(i,j)];
    }
    /** set the value. */
    public void set(int i, int j, double value) {
      storageD[ix.setDirect(i,j)] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 3. */
  public static class D3 extends ArrayDouble {
    private Index3D ix;
    /** Constructor for array of shape {len0,len1,len2}. */
    public D3 (int len0, int len1, int len2) {
      super(new int [] {len0, len1, len2});
      ix = (Index3D) indexCalc;
    }
    private D3 (Index i, double[] store) {
      super(i, store);
      ix = (Index3D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j, int k) {
      return storageD[ix.setDirect(i,j,k)];
    }
    /** set the value. */
    public void set(int i, int j, int k, double value) {
      storageD[ix.setDirect(i,j,k)] = value;
    }

    public IF getIF() { return new IF(); }
    public class IF {
      private int currElement = -1;
      private int size = (int) ix.getSize();
      public boolean hasNext(int howMany) { return currElement < size-howMany; }
      public double getNext() { return storageD[++currElement]; }
      public void setNext(double val) { storageD[++currElement] = val; }
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 4. */
  public static class D4 extends ArrayDouble {
    private Index4D ix;
    /** Constructor for array of shape {len0,len1,len2,len3}. */
    public D4 (int len0, int len1, int len2, int len3) {
      super(new int [] {len0, len1, len2, len3});
      ix = (Index4D) indexCalc;
    }
    private D4 (Index i, double[] store) {
      super(i, store);
      ix = (Index4D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j, int k, int l) {
      return storageD[ix.setDirect(i,j,k,l)];
    }
    /** set the value. */
    public void set(int i, int j, int k, int l, double value) {
      storageD[ix.setDirect(i,j,k,l)] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 5. */
  public static class D5 extends ArrayDouble {
    private Index5D ix;
    /** Constructor for array of shape {len0,len1,len2,len3,len4}. */
    public D5 (int len0, int len1, int len2, int len3, int len4) {
      super(new int [] {len0, len1, len2, len3, len4});
      ix = (Index5D) indexCalc;
    }
    private D5 (Index i, double[] store) {
      super(i, store);
      ix = (Index5D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j, int k, int l, int m) {
      return storageD[ix.setDirect(i,j,k,l, m)];
    }
    /** set the value. */
    public void set(int i, int j, int k, int l, int m, double value) {
      storageD[ix.setDirect(i,j,k,l, m)] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 6. */
  public static class D6 extends ArrayDouble {
    private Index6D ix;
    /** Constructor for array of shape {len0,len1,len2,len3,len4,len5,}. */
    public D6 (int len0, int len1, int len2, int len3, int len4, int len5) {
      super(new int [] {len0, len1, len2, len3, len4, len5});
      ix = (Index6D) indexCalc;
    }
    private D6 (Index i, double[] store) {
      super(i, store);
      ix = (Index6D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j, int k, int l, int m, int n) {
      return storageD[ix.setDirect(i,j,k,l,m,n)];
    }
    /** set the value. */
    public void set(int i, int j, int k, int l, int m, int n, double value) {
      storageD[ix.setDirect(i,j,k,l,m,n)] = value;
    }
  }

  /** Concrete implementation of Array specialized for doubles, rank 7. */
  public static class D7 extends ArrayDouble {
    private Index7D ix;
    /** Constructor for array of shape {len0,len1,len2,len3,len4,len5,len6}. */
    public D7 (int len0, int len1, int len2, int len3, int len4, int len5, int len6) {
      super(new int [] {len0, len1, len2, len3, len4, len5, len6});
      ix = (Index7D) indexCalc;
    }
    private D7 (Index i, double[] store) {
      super(i, store);
      ix = (Index7D) indexCalc;
    }
    /** get the value. */
    public double get(int i, int j, int k, int l, int m, int n, int o) {
      return storageD[ix.setDirect(i,j,k,l,m,n,o)];
    }
    /** set the value. */
    public void set(int i, int j, int k, int l, int m, int n, int o, double value) {
      storageD[ix.setDirect(i,j,k,l,m,n,o)] = value;
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.point;

import java.io.IOException;
import java.util.Formatter;
import java.util.List;
import javax.annotation.Nonnull;
import ucar.ma2.DataType;
import ucar.ma2.StructureData;
import ucar.ma2.StructureMembers;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Structure;
import ucar.nc2.Variable;
import ucar.nc2.VariableSimpleIF;
import ucar.nc2.constants.CDM;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.StructureDS;
import ucar.nc2.dataset.StructurePseudoDS;
import ucar.nc2.ft.DsgFeatureCollection;
import ucar.nc2.ft.PointFeature;
import ucar.nc2.ft2.coverage.TimeHelper;
import ucar.nc2.time.Calendar;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateFormatter;
import ucar.nc2.time.CalendarDateUnit;
import ucar.nc2.units.SimpleUnit;
import ucar.unidata.geoloc.EarthLocation;
import ucar.unidata.geoloc.EarthLocationImpl;
import ucar.unidata.geoloc.LatLonPoint;
import ucar.unidata.geoloc.LatLonPointImpl;
import ucar.unidata.geoloc.LatLonRect;
import ucar.unidata.geoloc.Station;
import ucar.unidata.geoloc.StationImpl;

/**
 * Helper class for using the netcdf-3 record dimension.
 *
 * @author caron
 * @since Feb 29, 2008
 */

public class RecordDatasetHelper {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(RecordDatasetHelper.class);

  protected NetcdfDataset ncfile;
  protected String obsTimeVName, nomTimeVName;
  protected String latVName, lonVName, zcoordVName, zcoordUnits;

  protected String stnIdVName, stnIndexVName, stnDescVName;
  protected StationHelper stationHelper;
  protected DataType stationIdType;

  protected StructureDS recordVar;
  protected Dimension obsDim;

  protected LatLonRect boundingBox;
  protected double minDate, maxDate;
  protected CalendarDateUnit timeUnit;

  protected double altScaleFactor = 1.0;

  protected Formatter errs = null;

  /**
   * Constructor.
   *
   * @param ncfile             the netccdf file
   * @param typedDataVariables list of data variables; all record variables will be added to this list, except . You
   *                           can remove extra
   * @param obsTimeVName       observation time variable name (required)
   * @param nomTimeVName       nominal time variable name (may be null)
   * @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
   */
  public RecordDatasetHelper(NetcdfDataset ncfile, String obsTimeVName, String nomTimeVName, List<VariableSimpleIF> typedDataVariables,
                             String recDimName, Formatter errBuffer) {
    this.ncfile = ncfile;
    this.obsTimeVName = obsTimeVName;
    this.nomTimeVName = nomTimeVName;
    this.errs = errBuffer;

    // check if we already have a structure vs if we have to add it.

    if (this.ncfile.hasUnlimitedDimension()) {
      this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
      this.obsDim = ncfile.getUnlimitedDimension();

    } else {
      if (recDimName == null)
        throw new IllegalArgumentException("File <" + this.ncfile.getLocation() +
                "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
      this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
      this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
    }

    // create member variables
    List<Variable> recordMembers = ncfile.getVariables();
    for (Variable v : recordMembers) {
      if (v == recordVar) continue;
      if (v.isScalar()) continue;
      if (v.getDimension(0) == this.obsDim)
        typedDataVariables.add(v);
    }

    // need the time units
    Variable timeVar = ncfile.findVariable(obsTimeVName);
    String timeUnitString = ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
    Calendar cal = TimeHelper.getCalendarFromAttribute(timeVar);

    try {
      timeUnit = CalendarDateUnit.withCalendar(cal, timeUnitString);

    } catch (Exception e) {
      if (null != errs)
        errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
      timeUnit = CalendarDateUnit.unixDateUnit;
    }
  }

  /**
   * Set extra information used by station obs datasets.
   * Use stnIdVName or stnIndexVName.
   *
   * @param stnIdVName   the obs variable that is used to find the station in the stnHash; may be type  int or a String (char).
   * @param stnDescVName optional station var containing station description
   */
  public void setStationInfo(String stnIdVName, String stnDescVName, String stnIndexVName, StationHelper stationHelper) {
    this.stnIdVName = stnIdVName;
    this.stnDescVName = stnDescVName;
    this.stnIndexVName = stnIndexVName;
    this.stationHelper = stationHelper;

    if (stnIdVName != null) {
      Variable stationVar = ncfile.findVariable(stnIdVName);
      stationIdType = stationVar.getDataType();
    }
  }

  public void setLocationInfo(String latVName, String lonVName, String zcoordVName) {
    this.latVName = latVName;
    this.lonVName = lonVName;
    this.zcoordVName = zcoordVName;

    // check for meter conversion
    if (zcoordVName != null) {
      Variable v = ncfile.findVariable(zcoordVName);
      zcoordUnits = ncfile.findAttValueIgnoreCase(v, CDM.UNITS, null);
      if (zcoordUnits != null)
        try {
          altScaleFactor = getMetersConversionFactor(zcoordUnits);
        } catch (Exception e) {
          if (errs != null) errs.format("%s", e.getMessage());
        }
    }
  }

  // make structure variable names to shortNames so StructureData sdata can
  // access it members
  public void setShortNames(String latVName, String lonVName, String altVName, String obsTimeVName, String nomTimeVName) {
    this.latVName = latVName;
    this.lonVName = lonVName;
    this.zcoordVName = altVName;
    this.obsTimeVName = obsTimeVName;
    this.nomTimeVName = nomTimeVName;
  }

  protected static double getMetersConversionFactor(String unitsString) throws Exception {
    SimpleUnit unit = SimpleUnit.factoryWithExceptions(unitsString);
    return unit.convertTo(1.0, SimpleUnit.meterUnit);
  }

  public Structure getRecordVar() {
    return (this.recordVar);
  }

  public int getRecordCount() {
    Dimension unlimitedDim = ncfile.getUnlimitedDimension();
    return unlimitedDim.getLength();
  }

  public void setTimeUnit(CalendarDateUnit timeUnit) {
    this.timeUnit = timeUnit;
  }

  public CalendarDateUnit getTimeUnit() {
    return this.timeUnit;
  }

  public LatLonPoint getLocation(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    double lat = sdata.convertScalarDouble(members.findMember(latVName));
    double lon = sdata.convertScalarDouble(members.findMember(lonVName));
    return new LatLonPointImpl(lat, lon);
  }

  public double getLatitude(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return sdata.convertScalarDouble(members.findMember(latVName));
  }

  public double getLongitude(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return sdata.convertScalarDouble(members.findMember(lonVName));
  }

  public double getZcoordinate(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return (zcoordVName == null) ? Double.NaN : sdata.convertScalarDouble(members.findMember(zcoordVName));
  }

  public String getZcoordUnits() {
    return zcoordUnits;
  }

  public CalendarDate getObservationTimeAsDate(StructureData sdata) {
    return timeUnit.makeCalendarDate(getObservationTime(sdata));
  }

  public double getObservationTime(StructureData sdata) {
    return getTime(sdata.findMember(obsTimeVName), sdata);
  }

  private double getTime(StructureMembers.Member timeVar, StructureData sdata) {
    if (timeVar == null) return 0.0;

    if ((timeVar.getDataType() == DataType.CHAR) || (timeVar.getDataType() == DataType.STRING)) {
      String time = sdata.getScalarString(timeVar);
      CalendarDate date = CalendarDateFormatter.isoStringToCalendarDate(null, time);
      if (date == null) {
        log.error("Cant parse date - not ISO formatted, = " + time);
        return 0.0;
      }
      return date.getMillis() / 1000.0;

    } else {
      return sdata.convertScalarDouble(timeVar);
    }
  }

  /*
   * This reads through all the records in the dataset, and constructs a list of
   * RecordPointObs or RecordStationObs. It does not cache the data.
   * <p>If stnIdVName is not null, its a StationDataset, then construct a Station HashMap of StationImpl
   * objects. Add the RecordStationObs into the list of obs for that station.
   *
   * @param cancel allow user to cancel
   * @return List of RecordPointObs or RecordStationObs
   * @throws IOException on read error
   *
  public List<RecordPointObs> readAllCreateObs(CancelTask cancel) throws IOException {

    // see if its a station or point dataset
    boolean hasStations = stnIdVName != null;
    if (hasStations)
      stnHash = new HashMap<Object, Station>();

    // get min and max date and lat,lon
    double minDate = Double.MAX_VALUE;
    double maxDate = -Double.MAX_VALUE;

    double minLat = Double.MAX_VALUE;
    double maxLat = -Double.MAX_VALUE;

    double minLon = Double.MAX_VALUE;
    double maxLon = -Double.MAX_VALUE;

    // read all the data, create a RecordObs
    StructureMembers members = null;
    List<RecordPointObs> records = new ArrayList<RecordPointObs>();
    int recno = 0;
    Structure.Iterator ii = recordVar.getStructureIterator();
    while (ii.hasNext()) {
      StructureData sdata = ii.next();
      if (members == null)
        members = sdata.getStructureMembers();

      Object stationId = null;
      if (hasStations) {
        if (stationIdType == DataType.INT) {
          stationId = sdata.getScalarInt(stnIdVName);
        } else
          stationId = sdata.getScalarString(stnIdVName).trim();
      }

      String desc = (stnDescVName == null) ? null : sdata.getScalarString(stnDescVName);
      double lat = sdata.getScalarDouble(latVName);
      double lon = sdata.getScalarDouble(lonVName);
      double alt = (altVName == null) ? 0.0 : altScaleFactor * sdata.getScalarDouble(altVName);
      double obsTime = sdata.convertScalarDouble(members.findMember(obsTimeVName));
      double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble(members.findMember(nomTimeVName));

      //double obsTime = sdata.convertScalarDouble( members.findMember( obsTimeVName) );
      //double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble( members.findMember( nomTimeVName));

      if (hasStations) {
        Station stn = stnHash.get(stationId);
        if (stn == null) {
          stn = new Station(stationId.toString(), desc, lat, lon, alt);
          stnHash.put(stationId, stn);
        }
        RecordStationObs stnObs = new RecordStationObs(stn, obsTime, nomTime, timeUnit, recno);
        records.add(stnObs);
        //stn.addObs( stnObs);

      } else {
        records.add(new RecordPointObs(new EarthLocation(lat, lon, alt), obsTime, nomTime, timeUnit, recno));
      }

      // track date range and bounding box
      minDate = Math.min(minDate, obsTime);
      maxDate = Math.max(maxDate, obsTime);

      minLat = Math.min(minLat, lat);
      maxLat = Math.max(maxLat, lat);
      minLon = Math.min(minLon, lon);
      maxLon = Math.max(maxLon, lon);

      recno++;
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    boundingBox = new LatLonRect(new LatLonPointImpl(minLat, minLon), new LatLonPointImpl(maxLat, maxLon));

    return records;
  }

  /* private boolean debugBB = false;
  public List getData(ArrayList records, LatLonRect boundingBox, CancelTask cancel) throws IOException {
    if (debugBB) System.out.println("Want bb= "+boundingBox);
    ArrayList result = new ArrayList();
    for (int i = 0; i < records.size(); i++) {
      RecordDatasetHelper.RecordPointObs r =  (RecordDatasetHelper.RecordPointObs) records.get(i);
      if (boundingBox.contains(r.getLatLon())) {
        if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
        result.add( r);
      }
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    return result;
  }

  // return List<PointObsDatatype>
  public List getData(ArrayList records, LatLonRect boundingBox, double startTime, double endTime, CancelTask cancel) throws IOException {
    if (debugBB) System.out.println("Want bb= "+boundingBox);
    ArrayList result = new ArrayList();
    for (int i = 0; i < records.size(); i++) {
      RecordDatasetHelper.RecordPointObs r =  (RecordDatasetHelper.RecordPointObs) records.get(i);
      if (boundingBox.contains(r.getLatLon())) {
        if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
        double timeValue = r.getObservationTime();
        if ((timeValue >= startTime) && (timeValue <= endTime))
          result.add( r);
      }
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    return result;
  }  */

  //////////////////////////////////////////////////////////////////////////////////////
  public PointFeature factory(StationImpl s, StructureData sdata, int recno) {
    if (s == null)
      return new RecordPointObs(sdata, recno);
    else
      return new RecordStationObs(s, sdata, recno);
  }

  class RecordPointObs extends PointFeatureImpl {
    protected int recno;
    protected StructureData sdata;

    RecordPointObs(int recno) {
      super(null, RecordDatasetHelper.this.timeUnit);
      this.recno = recno;
    }

    // Constructor for the case where you keep track of the location, time of each record, but not the data.
    protected RecordPointObs(DsgFeatureCollection dsg, EarthLocation location, double obsTime, double nomTime, CalendarDateUnit timeUnit, int recno) {
      super(dsg, location, obsTime, nomTime, timeUnit);
      this.recno = recno;
    }

    // Constructor for when you already have the StructureData and want to wrap it in a StationObsDatatype
    protected RecordPointObs(StructureData sdata, int recno) {
      super(null, RecordDatasetHelper.this.timeUnit);
      this.sdata = sdata;
      this.recno = recno;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);

      // this assumes the lat/lon/alt is stored in the obs record
      double lat = sdata.convertScalarDouble(members.findMember(latVName));
      double lon = sdata.convertScalarDouble(members.findMember(lonVName));
      double alt = (zcoordVName == null) ? 0.0 : altScaleFactor * sdata.convertScalarDouble(members.findMember(zcoordVName));
      location = new EarthLocationImpl(lat, lon, alt);
    }

    public String getId() {
      return Integer.toString(recno);
    }

    public LatLonPoint getLatLon() {
      return new LatLonPointImpl(location.getLatitude(), location.getLongitude());
    }

    @Nonnull
    public StructureData getFeatureData() throws IOException {
      if (null == sdata) {
        try {
          // deal with files that are updating // LOOK kludge?
          if (recno > getRecordCount()) {
            int n = getRecordCount();
            ncfile.syncExtend();
            log.info("RecordPointObs.getData recno=" + recno + " > " + n + "; after sync= " + getRecordCount());
          }

          sdata = recordVar.readStructure(recno);
        } catch (ucar.ma2.InvalidRangeException e) {
          e.printStackTrace();
          throw new IOException(e.getMessage());
        }
      }
      return sdata;
    }

    @Nonnull
    public ucar.ma2.StructureData getDataAll() throws java.io.IOException {
      return getFeatureData();
    }

  }

  //////////////////////////////////////////////////////////////////////////////////////

  // a PointObs with the location info stored as a Station
  class RecordStationObs extends RecordPointObs {
    private Station station;

    /**
     * Constructor for the case where you keep track of the station, time of each record, but the data reading is deferred.
     *
     * @param station data is for this Station
     * @param obsTime observation time
     * @param nomTime nominal time (may be NaN)
     * @param recno   data is at this record number
     */
    protected RecordStationObs(DsgFeatureCollection dsg, Station station, double obsTime, double nomTime, CalendarDateUnit timeUnit, int recno) {
      super(dsg, station, obsTime, nomTime, timeUnit, recno);
      this.station = station;
    }

    // Constructor for when you have everything
    protected RecordStationObs(Station station, double obsTime, double nomTime, StructureData sdata, int recno) {
      super(recno);
      this.station = station;
      this.location = station;
      this.obsTime = obsTime;
      this.nomTime = nomTime;
      this.sdata = sdata;
    }

    // Constructor for when you already have the StructureData and Station, and calculate times
    protected RecordStationObs(Station station, StructureData sdata, int recno) {
      super(recno);
      this.station = station;
      this.location = station;
      this.sdata = sdata;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);
    }

    // Constructor for when you already have the StructureData, and need to find Station and times
    protected RecordStationObs(StructureData sdata, int recno, boolean useId) {
      super(recno);
      this.recno = recno;
      this.sdata = sdata;
      this.timeUnit = RecordDatasetHelper.this.timeUnit;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);

      if (useId) {
        // this assumes the station id/name is stored in the obs record
        String stationId;
        if (stationIdType == DataType.INT) {
          stationId = Integer.toString(sdata.getScalarInt(stnIdVName));
        } else
          stationId = sdata.getScalarString(stnIdVName).trim();
        station = stationHelper.getStation(stationId);
        if (null != errs) errs.format(" cant find station id = <%s> when reading record %d%n", stationId, recno);
        log.error(" cant find station id = <" + stationId + "> when reading record " + recno);

      } else {
        // use a station index
        List<Station> stations = stationHelper.getStations();
        int stationIndex = sdata.getScalarInt(stnIndexVName);
        if (stationIndex < 0 || stationIndex >= stations.size()) {
          if (null != errs)
            errs.format(" cant find station at index =%d when reading record %d%n", stationIndex, recno);
          log.error("cant find station at index = " + stationIndex + " when reading record " + recno);
        } else
          station = stations.get(stationIndex);
      }

      location = station;
    }

  }

}