/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.cdm.nc2;

import dap4.cdm.NodeMap;
import dap4.core.data.DSP;
import dap4.core.data.DataCursor;
import dap4.core.dmr.*;
import dap4.core.util.DapException;
import dap4.core.util.DapUtil;
import dap4.core.util.Index;
import dap4.core.util.Odometer;
import ucar.ma2.Array;
import ucar.nc2.Attribute;
import ucar.nc2.Group;
import ucar.nc2.Variable;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Create a set of CDM ucar.ma2.array objects that wrap a DSP.
 */

public class DataToCDM
{
    static public boolean DEBUG = false;

    //////////////////////////////////////////////////
    // Constants

    static protected final int COUNTSIZE = 8; // databuffer as specified by the DAP4 spec

    static protected final String LBRACE = "{";
    static protected final String RBRACE = "}";

    //////////////////////////////////////////////////
    // Instance variables

    protected DapNetcdfFile ncfile = null;
    protected DSP dsp = null;
    protected DapDataset dmr = null;
    protected Group cdmroot = null;
    protected Map<Variable, Array> arraymap = null;
    protected NodeMap nodemap = null;

    //////////////////////////////////////////////////
    //Constructor(s)

    /**
     * Constructor
     *
     * @param ncfile the target NetcdfDataset
     * @param dsp    the compiled D4 databuffer
     */

    public DataToCDM(DapNetcdfFile ncfile, DSP dsp, NodeMap nodemap)
            throws DapException
    {
        this.ncfile = ncfile;
        this.dsp = dsp;
        this.dmr = dsp.getDMR();
        this.nodemap = nodemap;
        this.cdmroot = ncfile.getRootGroup();
        arraymap = new HashMap<Variable, Array>();
        // Add endianness attribute to the group
       /* ByteOrder remoteorder = ncfile.getDSP().getOrder();
        String endianness = null;
        if(remoteorder != null) {
            if(remoteorder == ByteOrder.BIG_ENDIAN)
                endianness = "big";
            else if(remoteorder == ByteOrder.BIG_ENDIAN)
                endianness = "little";
        }
        if(endianness != null) {
            Attribute aendian = new Attribute(DapUtil.ENDIANATTRNAME, endianness);
            this.cdmroot.addAttribute(aendian);
        } */
    }

    //////////////////////////////////////////////////
    // Compile DataCursor objects to ucar.ma2.Array objects

    /* package access */
    Map<Variable, Array>
    create()
            throws DapException
    {
        // iterate over the variables represented in the DSP
        List<DapVariable> topvars = this.dmr.getTopVariables();
        Map<Variable, Array> map = null;
        for(DapVariable var : topvars) {
            DataCursor cursor = this.dsp.getVariableData(var);
            Array array = createVar(cursor);
            Variable cdmvar = (Variable) nodemap.get(var);
            arraymap.put(cdmvar, array);
        }
        return this.arraymap;
    }

    protected Array
    createVar(DataCursor data)
            throws DapException
    {
        Array array = null;
        DapVariable d4var = (DapVariable) data.getTemplate();
        switch (d4var.getBaseType().getTypeSort()) {
        default: // atomic var
            array = createAtomicVar(data);
            break;
        case Sequence:
            array = createSequence(data);
            break;
        case Structure:
            array = createStructure(data);
            break;
        }
        if(d4var.isTopLevel() && this.dsp.getChecksumMode().enabled(dsp.getChecksumMode())) {
            // transfer the checksum attribute
            int csum = d4var.getChecksum();
            String scsum = String.format("0x%08x", csum);
            Variable cdmvar = (Variable) nodemap.get(d4var);
            Attribute acsum = new Attribute(DapUtil.CHECKSUMATTRNAME, scsum);
            cdmvar.addAttribute(acsum);
        }
        return array;
    }

    /**
     * Create an Atomic Valued variable.
     *
     * @return An Array object wrapping d4var.
     * @throws DapException
     */
    protected CDMArrayAtomic
    createAtomicVar(DataCursor data)
            throws DapException
    {
        CDMArrayAtomic array = new CDMArrayAtomic(data);
        return array;
    }

    /**
     * Create an array of structures. WARNING: the underlying CDM code
     * (esp. NetcdfDataset) apparently does not support nested
     * structure arrays; so this code may throw an exception.
     *
     * @return A CDMArrayStructure for the databuffer for this struct.
     * @throws DapException
     */
    protected CDMArrayStructure
    createStructure(DataCursor data)
            throws DapException
    {
        CDMArrayStructure arraystruct = new CDMArrayStructure(this.cdmroot, data);
        DapVariable var = (DapVariable) data.getTemplate();
        DapStructure struct = (DapStructure) var.getBaseType();
        int nmembers = struct.getFields().size();
        List<DapDimension> dimset = var.getDimensions();
        Odometer odom = Odometer.factory(DapUtil.dimsetToSlices(dimset));
        while(odom.hasNext()) {
            Index index = odom.next();
            long offset = index.index();
            DataCursor[] cursors = (DataCursor[]) data.read(index);
            DataCursor ithelement = cursors[0];
            for(int f = 0; f < nmembers; f++) {
                DataCursor dc = (DataCursor) ithelement.readField(f);
                Array afield = createVar(dc);
                arraystruct.add(offset, f, afield);
            }
        }
        return arraystruct;
    }

    /**
     * Create a sequence. WARNING: the underlying CDM code
     * (esp. NetcdfDataset) apparently does not support nested
     * sequence arrays.
     *
     * @param data the data underlying this sequence instance
     * @return A CDMArraySequence for this instance
     * @throws DapException
     */

    protected CDMArraySequence
    createSequence(DataCursor data)
            throws DapException
    {
        CDMArraySequence arrayseq = new CDMArraySequence(this.cdmroot, data);
        DapVariable var = (DapVariable) data.getTemplate();
        DapSequence template = (DapSequence) var.getBaseType();
        List<DapDimension> dimset = var.getDimensions();
        long dimsize = DapUtil.dimProduct(dimset);
        int nfields = template.getFields().size();
        Odometer odom = Odometer.factory(DapUtil.dimsetToSlices(dimset));
        while(odom.hasNext()) {
            odom.next();
            DataCursor seq = ((DataCursor[]) data.read(odom.indices()))[0];
            long nrecords = seq.getRecordCount();
            for(int r = 0; r < nrecords; r++) {
                DataCursor rec = seq.readRecord(r);
                for(int f = 0; f < nfields; f++) {
                    DataCursor dc = rec.readField(f);
                    Array afield = createVar(dc);
                    arrayseq.add(r, f, afield);
                }
            }
        }
        return arrayseq;
    }
}


/* Copyright 2016, University Corporation for Atmospheric Research
   See the LICENSE.txt file for more information.
*/

package thredds.server.reify;

import org.apache.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import thredds.core.TdsRequestedDataset;
import ucar.httpservices.HTTPUtil;
import ucar.nc2.FileWriter2;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFileWriter;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.iosp.NCheader;
import ucar.nc2.iosp.hdf5.H5header;
import ucar.nc2.iosp.netcdf3.N3header;
import ucar.nc2.util.CancelTaskImpl;
import ucar.nc2.write.Nc4Chunking;
import ucar.nc2.write.Nc4ChunkingDefault;
import ucar.unidata.io.RandomAccessFile;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;

import static  ucar.nc2.iosp.NCheader.*;

/**
 * Local File Materialization
 * for server-side computing.
 * The tag for this controller is .../download/* or .../restrictedAccess/download
 *

 * The goal for this controller is to allow external code to have the
 * thredds server materialize a dataset into the file system .
 *

 * Note that this functionality is not strictly necessary
 * since it could all be done on the client side.
 * Since the server is doing all of the work of converting
 * a dataset to a file, the client does not need to have
 * its own code for this. This means:
 * 1. It is lightweight WRT the client
 * 2. It is language independent
 *

 * Assumptions:
 * 1. The external code has authenticated to the thredds server
 * using some authentication mechanism such as
 * client-side certificates or tomcat roles. This code performs
 * no authentication.
 *

 * 2. The external code is running on the same machine as the thredds
 * server, or at least has a common file system so that file system
 * writes by thredds are visible to the external code.
 *

 * WARNING: In order to support downloading of non-file datasets
 * (e.g. via DAP2 or DAP4) this controller will
 * re-call the server to obtain the output of the request. Experimentation
 * shows this is not a problem. Circularity is tested to ensure
 * the no download loop occurs.
 *

 * Issues:
 * 1. What file formats are allowed for materialized datasets?
 * - Currently only netcdf-3 (classic) and netcdf-4 (enhanced).
 *

 * A set of query parameters control the operation of this servlet.
 * Note that all of the query parameter values (but not keys) are
 * assumed to be url-encoded (%xx), so beware.
 * Also, all return values are url-encoded.
 * Using query parameters means that it is possible to use an Http Form
 * to send them; see TestDownload.
 *

 * Download Request
 * ----------------
 * Download parameters:
 * - request=download -- materialize a file into the download directory
 * - format=netcdf3|netcdfd4 -- specify the download format
 * - url={a thredds server dataset access url} -- specify the actual dataset.
 * - target={path of the downloaded file} -- if it already exists, then it
 * will be overwritten.
 * Notes:
 * 1. the host, port and servlet prefix of the url will be
 * ignored and replaced with the "<host>+<port>/thredds" of the thredds
 * server. This is to prevent attempts to use the thredds server to access
 * external data sources, which would otherwise provide a security leak.
 * 2. The target path must be a relative path. It is interpreted as relative
 * to the value of a -Dtds.download.dir java flag, which must specify
 * an absolute path to a directory into which the downloads are stored.
 * Assuming the Tomcat server is being used, then the path must be
 * writeable by user tomcat, which means that its owner will end up
 * being user tomcat or at least tomcat must be in the group assigned to
 * this directory.
 *

 * Return value: download=<absolute path to which file was downloaded>
 *

 * Misc Parameters:
 * - testinfo={testing info} -- for testing purposes
 *

 * Inquire Request
 * ----------------
 * Download parameters:
 * - request=inquire -- Inquire about various parameters
 * - inquire={semicolon separated arguments} -- Inquire about the default download dir
 *

 * Return value: <key>={value of the requested key}; one key-value pair per line.
 */

@Controller
@RequestMapping(value = {"/download", "/restrictedAccess/download"})
public class DownloadController extends LoadCommon
{
    //////////////////////////////////////////////////
    // Constants

    static final protected boolean DEBUG = true;

    static final protected String DEFAULTREQUESTNAME = "download";

    static final protected String DEFAULTDOWNLOADDIR = "download";

    static final protected String FILESERVERSERVLET = "/fileServer/";

    //////////////////////////////////////////////////
    // Instance variables

    protected Nc4Chunking.Strategy strategy = Nc4Chunking.Strategy.standard;
    protected Nc4Chunking chunking = new Nc4ChunkingDefault();
    protected DownloadParameters params = null;
    protected String downloadform = null;

    //////////////////////////////////////////////////
    // Constructor(s)

    public DownloadController()
            throws ServletException
    {
        super();
    }

    //////////////////////////////////////////////////

    /**
     * Invoked once on first request so that everything is available,
     * especially Spring stuff.
     */
    public void doonce(HttpServletRequest req)
            throws SendError
    {
        if(once)
            return;
        super.initOnce(req);
        if(this.downloaddir == null)
            throw new SendError(HttpStatus.SC_PRECONDITION_FAILED, "Download disabled");
        this.downloaddirname = new File(this.downloaddir).getName();

        // Get the download form
        File downform = null;
        downform = tdsContext.getDownloadForm();
        if(downform == null) {   // Look in WEB-INF directory
            File root = tdsContext.getServletRootDirectory();
            downform = new File(root, DEFAULTDOWNLOADFORM);
        }
        try {
            this.downloadform = loadForm(downform);
        } catch (IOException ioe) {
            throw new SendError(HttpStatus.SC_PRECONDITION_FAILED, ioe);
        }
    }

    // Setup for each request
    public void setup(HttpServletRequest req, HttpServletResponse resp)
            throws SendError
    {
        this.req = req;
        this.res = resp;
        if(!once)
            doonce(req);

        // Parse any query parameters
        try {
            this.params = new DownloadParameters(req);
        } catch (IOException ioe) {
            throw new SendError(res.SC_BAD_REQUEST, ioe);
        }
    }

    //////////////////////////////////////////////////
    // Controller entry point(s)

    @RequestMapping(value = "**", method = RequestMethod.GET)
    public void
    doGet(HttpServletRequest req, HttpServletResponse res)
            throws ServletException
    {
        try {
            setup(req, res);
            String sresult = null;
            switch (this.params.command) {
            case DOWNLOAD:
                try {
                    String fulltargetpath = download();
                    if(this.params.fromform) {
                        sendForm("Download succeeded: result file: " + fulltargetpath);
                    } else {
                        Map<String, String> result = new HashMap<>();
                        result.put("download", fulltargetpath);
                        sresult = mapToString(result, true, "download");
                        sendOK(sresult);
                    }
                } catch (SendError se) {
                    if(this.params.fromform) {
                        // Send back the download form with error msg
                        sendForm("Download failed: " + se.getMessage());
                    } else
                        throw se;
                }
                break;
            case INQUIRE:
                sresult = inquire();
                // Send back the inquiry answers
                sendOK(sresult);
                break;
            case NONE: // Use form-based download
                // Send back the download form
                sendForm("No files downloaded");
                break;
            }
        } catch (SendError se) {
            sendError(se);
        } catch (Exception e) {
            String msg = getStackTrace(e);
            sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg, e);
        }
    }

    //////////////////////////////////////////////////

    /**
     * @return absolute path to the downloaded file
     * @throws SendError if bad request
     */
    protected String
    download()
    {
        String target = this.params.target;

        if(target == null)
            throw new SendError(HttpServletResponse.SC_BAD_REQUEST, "No target specified");

        // Make sure target path does not contain '..'
        if(target.indexOf("..") >= 0)
            throw new SendError(HttpServletResponse.SC_BAD_REQUEST, "Target parameter contains '..': " + target);

        // See if the target is relative or absolute
        File ftarget = new File(target);
        if(ftarget.isAbsolute())
            throw new SendError(HttpServletResponse.SC_BAD_REQUEST, "Target parameter must be a relative path: " + target);

        // See if we have a download dir
        if(this.downloaddir == null)
            throw new SendError(HttpServletResponse.SC_BAD_REQUEST, "No download directory specified for relative target: " + target);

        // Make target relative to download dir and convert to absolute path
        StringBuilder b = new StringBuilder();
        b.append(this.downloaddir);
        b.append('/');
        b.append(target);
        ftarget = new File(b.toString());
        String fulltarget = HTTPUtil.canonicalpath(ftarget.getAbsolutePath());

        // Make sure that all intermediate directories exist
        File parent = ftarget.getParentFile();
        if(!parent.exists() && !ftarget.getParentFile().mkdirs())
            throw new SendError(HttpServletResponse.SC_FORBIDDEN, "Target file parent directory cannot be created: " + fulltarget);

        // If file exists, delete it if requested
        if(!this.params.overwrite && ftarget.exists())
            throw new SendError(HttpServletResponse.SC_FORBIDDEN, "Target file exists and overwrite is not set");
        else if(this.params.overwrite && ftarget.exists()) {
            if(!ftarget.delete())
                throw new SendError(HttpServletResponse.SC_FORBIDDEN, "Target file exists and cannot be deleted");
        }

        String surl = this.params.url;
        URL url;
        try {
            url = new URL(surl);
        } catch (MalformedURLException mue) {
            throw new SendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed URL: " + surl);
        }

        // Make sure we are not recursing
        String path = url.getPath();
        if(path.toLowerCase().indexOf(this.requestname) >= 0)
            throw new SendError(HttpServletResponse.SC_FORBIDDEN,
                    String.format("URL is recursive on /%s: %s", this.requestname, path));

        // Make sure that path starts with "/thredds"
        if(!path.startsWith("/" + this.threddsname))
            throw new SendError(HttpServletResponse.SC_FORBIDDEN,
                    String.format("URL does not reference %s: %s",
                            this.threddsname, path));

        // Rebuild the url to keep it under our control
        b.setLength(0); // reuse
        b.append(url.getProtocol());
        b.append("://");
        b.append(this.server);
        b.append(path);
        String trueurl = b.toString();

        // Get NetcdfFile object
        // Now, shortcircuit requests for fileServer case
        boolean directcopy = false;
        if(path.indexOf(FILESERVERSERVLET) >= 0) {
            String truepath = directAccess(path, this.req, this.res);
            directcopy = canCopy(truepath, this.params.format);
            if(directcopy) try {
                Path src = new File(truepath).toPath();
                Path dst = new File(fulltarget).toPath();
                Files.deleteIfExists(dst);
                Files.copy(src, dst);
            } catch (IOException e) {
                throw new SendError(HttpServletResponse.SC_FORBIDDEN, truepath);
            }
        }
        if(!directcopy) {
            NetcdfFile ncfile = null;
            try {
                CancelTaskImpl cancel = new CancelTaskImpl();
                ncfile = NetcdfDataset.openFile(trueurl, cancel);
                switch (this.params.format) {
                case NETCDF3:
                    makeNetcdf3(ncfile, fulltarget);
                    break;
                case NETCDF4:
                    makeNetcdf4(ncfile, fulltarget);
                    break;
                default:
                    throw new SendError(HttpServletResponse.SC_NOT_IMPLEMENTED,
                            String.format("%s: return format %s not implemented",
                                    path, params.format.getName()));
                }
            } catch (IOException ioe) {
                throw new SendError(res.SC_BAD_REQUEST, ioe);
            }
        }
        // Return the absolute path of the target as the content
        return (fulltarget);
    }

    //////////////////////////////////////////////////
    // Reifiers

    protected void
    makeNetcdf4(NetcdfFile ncfile, String target)
            throws IOException
    {
        try {
            CancelTaskImpl cancel = new CancelTaskImpl();
            FileWriter2 writer = new FileWriter2(ncfile, target,
                    NetcdfFileWriter.Version.netcdf4,
                    chunking);
            writer.getNetcdfFileWriter().setLargeFile(true);
            NetcdfFile ncfileOut = writer.write(cancel);
            if(ncfileOut != null) ncfileOut.close();
            cancel.setDone(true);
        } catch (IOException ioe) {
            throw ioe; // temporary
        }
    }

    protected void
    makeNetcdf3(NetcdfFile ncfile, String target)
            throws IOException
    {
        try {
            CancelTaskImpl cancel = new CancelTaskImpl();
            FileWriter2 writer = new FileWriter2(ncfile, target,
                    NetcdfFileWriter.Version.netcdf3,
                    chunking);
            writer.getNetcdfFileWriter().setLargeFile(true);
            NetcdfFile ncfileOut = writer.write(cancel);
            if(ncfileOut != null) ncfileOut.close();
            cancel.setDone(true);
        } catch (IOException ioe) {
            throw ioe; // temporary
        }
    }

    //////////////////////////////////////////////////

    /**
     * Given a typical string, insert backslashes
     * before '"' and '\\' characters and control characters.
     */
    static protected String escapeString(String s)
    {
        StringBuilder buf = new StringBuilder();
        for(int i = 0; i < s.length(); i++) {
            int c = s.charAt(i);
            switch (c) {
            case '"':
                buf.append("\\\"");
                break;
            case '\\':
                buf.append("\\\\");
                break;
            case '\n':
                buf.append('\n');
                break;
            case '\r':
                buf.append('\r');
                break;
            case '\t':
                buf.append('\r');
                break;
            case '\f':
                buf.append('\f');
                break;
            default:
                if(c < ' ')
                    buf.append(String.format("\\x%02x", (c & 0xff)));
                else
                    buf.append((char) c);
                break;
            }
        }
        return buf.toString();
    }

    protected String
    directAccess(String relpath, HttpServletRequest req, HttpServletResponse res)
    {
        // Strip off /fileServer and everythihg before it
        int index = relpath.indexOf(FILESERVERSERVLET);
        assert index >= 0;
        relpath = relpath.substring(index + FILESERVERSERVLET.length(), relpath.length());
        relpath = HTTPUtil.abspath(HTTPUtil.canonicalpath(relpath));
        String realpath = TdsRequestedDataset.getLocationFromRequestPath(relpath);
        File f = new File(realpath);
        if(!f.exists() || !f.canRead())
            throw new SendError(res.SC_NOT_FOUND, "Not found: " + realpath);
        if(!TdsRequestedDataset.resourceControlOk(req, res, realpath))
            throw new SendError(res.SC_FORBIDDEN, "Permissions failure: " + realpath);
        return realpath;
    }

    protected boolean
    canCopy(String truepath, FileFormat targetformat)
    {
        try {
            RandomAccessFile raf = new RandomAccessFile(truepath, "r");
            int format = NCheader.checkFileType(raf);
            switch (format) {
            case NC_FORMAT_CLASSIC:
            case NC_FORMAT_64BIT_OFFSET:
                return targetformat == FileFormat.NETCDF3;
            case NC_FORMAT_NETCDF4:
            case NC_FORMAT_64BIT_DATA:
            case NC_FORMAT_HDF4:
                return targetformat == FileFormat.NETCDF4;
            default: break;
            }
        } catch (IOException e) {
            return false;
        }
        return false;
    }

    //////////////////////////////////////////////////

    @Override
    protected String
    buildForm(String msg)
    {
        StringBuilder svc = new StringBuilder();
        svc.append(this.server);
        svc.append("/");
        svc.append(this.threddsname);
        String form = String.format(this.downloadform,
                svc.toString(),
                msg
        );
        return form;
    }

}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.cdm;

import dap4.core.data.DataCursor;
import dap4.core.dmr.DapDimension;
import dap4.core.dmr.DapType;
import dap4.core.dmr.TypeSort;
import dap4.core.util.DapException;
import dap4.core.util.Index;
import dap4.core.util.Slice;
import ucar.ma2.ForbiddenConversionException;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;
import ucar.nc2.CDMNode;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.dataset.NetcdfDataset;

import java.util.ArrayList;
import java.util.List;

/**
 * CDM related Constants and utilities
 * common to client and server code
 */

abstract public class CDMUtil
{

    static final String hexchars = "0123456789abcdef";

    /**
     * Convert a list of ucar.ma2.Range to a list of Slice
     * More or less the inverst of create CDMRanges
     *
     * @param rangelist the set of ucar.ma2.Range
     * @result the equivalent list of Slice
     */
    static public List<Slice>
    createSlices(List<Range> rangelist)
            throws dap4.core.util.DapException
    {
        List<Slice> slices = new ArrayList<Slice>(rangelist.size());
        for(int i = 0; i < rangelist.size(); i++) {
            Range r = rangelist.get(i);
            // r does not store last
            int stride = r.stride();
            int first = r.first();
            int n = r.length();
            int stop = first + (n * stride);
            Slice cer = new Slice(first, stop - 1, stride);
            slices.add(cer);
        }
        return slices;
    }

    /**
     * Test a List<Range> against a List<DapDimension>
     * to see if the range list represents the whole
     * set of dimensions within the specified indices.
     *
     * @param rangelist the set of ucar.ma2.Range
     * @param dimset    the set of DapDimensions
     * @param start     start looking here
     * @param stop      stop looking here
     * @result true if rangelist is whole; false otherwise.
     */

    static public boolean
    isWhole(List<Range> rangelist, List<DapDimension> dimset, int start, int stop)
            throws dap4.core.util.DapException
    {
        int rsize = (rangelist == null ? 0 : rangelist.size());
        if(rsize != dimset.size())
            throw new dap4.core.util.DapException("range/dimset rank mismatch");
        if(rsize == 0)
            return true;
        if(start < 0 || stop < start || stop > rsize)
            throw new dap4.core.util.DapException("Invalid start/stop indices");

        for(int i = start; i < stop; i++) {
            Range r = rangelist.get(i);
            DapDimension d = dimset.get(i);
            if(r.stride() != 1 || r.first() != 0 || r.length() != d.getSize())
                return false;
        }
        return true;
    }

    /**
     * Test a List<Range> against a List<Slice>
     * to see if the range list is whole
     * wrt the slices
     *
     * @param rangelist the set of ucar.ma2.Range
     * @param slices    the set of slices
     * @result true if rangelist is whole wrt slices; false otherwise.
     */
    static public boolean
    isWhole(List<Range> rangelist, List<Slice> slices)
            throws dap4.core.util.DapException
    {
        if(rangelist.size() != slices.size())
            return false;
        for(int i = 0; i < rangelist.size(); i++) {
            Range r = rangelist.get(i);
            Slice slice = slices.get(i);
            if(r.stride() != 1 || r.first() != 0 || r.length() != slice.getCount())
                return false;
        }
        return true;
    }

    /**
     * Test a List<Range> against the CDM variable's dimensions
     * to see if the range list is whole
     * wrt the dimensions
     *
     * @param rangelist the set of ucar.ma2.Range
     * @param var       the cdm var
     * @result true if rangelist is whole wrt slices; false otherwise.
     */
    static public boolean
    isWhole(List<Range> rangelist, Variable var)
            throws dap4.core.util.DapException
    {
        List<Dimension> dimset = var.getDimensions();
        if(rangelist.size() != dimset.size())
            return false;
        for(int i = 0; i < rangelist.size(); i++) {
            Range r = rangelist.get(i);
            Dimension dim = dimset.get(i);
            if(r.stride() != 1 || r.first() != 0 || r.length() != dim.getLength())
                return false;
        }
        return true;
    }

    static public List<ucar.ma2.Range>
    createCDMRanges(List<Slice> slices)
            throws DapException
    {
        List<ucar.ma2.Range> cdmranges = new ArrayList<Range>();
        for(int i = 0; i < slices.size(); i++) {
            Slice r = slices.get(i);
            try {
                ucar.ma2.Range cmdr;
                cmdr = new ucar.ma2.Range((int) r.getFirst(),
                        (int) r.getLast(),
                        (int) r.getStride());
                cdmranges.add(cmdr);
            } catch (InvalidRangeException ire) {
                throw new DapException(ire);
            }
        }
        return cdmranges;
    }

    /**
     * NetcdfDataset can end up wrapping a variable
     * in multiple wrapping classes (e.g. VariableDS).
     * Goal of this procedure is to get down to the
     * lowest level Variable instance
     *
     * @param var possibly wrapped variable
     * @return the lowest level Variable instance
     */
    static public Variable unwrap(Variable var)
    {
/*        for(;;) {
            if(var instanceof VariableDS) {
                VariableDS vds = (VariableDS) var;
                var = vds.getOriginalVariable();
                if(var == null) {
                    var = vds;
                    break;
                }
            } else if(var instanceof StructureDS) {
                StructureDS sds = (StructureDS) var;
                var = sds.getOriginalVariable();
                if(var == null) {
                    var = sds;
                    break;
                }
            } else
                break;
        }
        return var;
        */
        return (Variable) CDMNode.unwrap(var);
    }

    /**
     * NetcdfDataset can wrap a NetcdfFile.
     * Goal of this procedure is to get down to the
     * lowest level NetcdfFile instance.
     *
     * @param file NetcdfFile or NetcdfDataset
     * @return the lowest level NetcdfFile instance
     */
    static public NetcdfFile unwrapfile(NetcdfFile file)
    {
        for(; ; ) {
            if(file instanceof NetcdfDataset) {
                NetcdfDataset ds = (NetcdfDataset) file;
                file = ds.getReferencedFile();
                if(file == null) break;
            } else break;
        }
        return file;
    }

    static public boolean
    hasVLEN(List<Range> ranges)
    {
        if(ranges == null || ranges.size() == 0) return false;
        return ranges.get(ranges.size() - 1) == Range.VLEN;
    }

    /**
     * Test if any dimension is variable length
     */
    static public boolean
    hasVLEN(Variable v)
    {
        return containsVLEN(v.getDimensions());
    }

    /**
     * Test if any dimension is variable length
     */
    static public boolean
    containsVLEN(List<Dimension> dimset)
    {
        if(dimset == null) return false;
        for(Dimension dim : dimset) {
            if(dim.isVariableLength())
                return true;
        }
        return false;
    }

    /**
     * Compute the shape inferred from a set of slices.
     * 'Effective' means that any trailing vlen will be
     * ignored.
     *
     * @param dimset from which to generate shape
     * @return
     */
    static public int[]
    computeEffectiveShape(List<DapDimension> dimset)
    {
        if(dimset == null || dimset.size() == 0)
            return new int[0];
        int effectiverank = dimset.size();
        int[] shape = new int[effectiverank];
        for(int i = 0; i < effectiverank; i++) {
            shape[i] = (int) dimset.get(i).getSize();
        }
        return shape;
    }

    /*
    static public int
    computeVariableSize(View view, DapVariable var, boolean scalar)
    {
	DapType dt = var.getBaseType();
        ViewVariable annotation = view.getAnnotation(var);
        int dimproduct = (scalar ? 1 : computeDimProduct(annotation.getSlices()));
        int elementsize = 0;
        switch (dt.getTypeSort()) {
	default: // atomic variable
            // This does not work for String or Opaque.
            elementsize =  CDMUtil.daptypeSize(dt.getTypeSort());
            break;
        case STRUCTURE:
        case SEQUENCE:
            for(DapVariable field : ((DapStructure) dt).getFields()) {
                elementsize += computeVariableSize(dataset, field, false);
            }
            break;

        default:
            break;
        }
        return dimproduct * elementsize;
    }
    */

    /**
     * Extract, as an object, value from a (presumably)
     * atomic typed array of values; dataset position
     * is presumed correct.
     *
     * @param atomtype type of object to extract ; must not be Enum
     * @param dataset  Data containing the objects
     * @param index    Which element of dataset to read
     * @return resulting value as an Object; value does not necessarily conform
     * to Convert.ValueClass.
     */

    /**
     * Extract, as a long, value from a (presumably)
     * atomic typed array of values; dataset position
     * is presumed correct.
     *
     * @param atomtype type of object to extract
     * @param dataset  Data containing the objects
     * @param index    Which element of dataset to read
     * @return resulting value as a long
     * @throws ForbiddenConversionException if cannot convert to long
     */

    static public long
    extractLongValue(TypeSort atomtype, DataCursor dataset, Index index)
            throws DapException
    {
        Object result;
        result = dataset.read(index);
        long lvalue = CDMTypeFcns.extract(atomtype, result);
        return lvalue;
    }

    /**
     * Extract, as a double, value from a (presumably)
     * atomic typed array of values; dataset position
     * is presumed correct.
     *
     * @param atomtype type of object to extract
     * @param dataset  Data containing the objects
     * @param index    Which element of dataset to read
     * @return resulting value as a double
     * @throws ForbiddenConversionException if cannot convert to double
     */

    static public double
    extractDoubleValue(TypeSort atomtype, DataCursor dataset, Index index)
            throws DapException
    {
        Object result;
        result = dataset.read(index);
        double dvalue = 0.0;
        if(atomtype.isIntegerType() || atomtype.isEnumType()) {
            long lvalue = extractLongValue(atomtype, dataset, index);
            dvalue = (double) lvalue;
        } else if(atomtype == TypeSort.Float32) {
            dvalue = (double) ((Float) result).floatValue();
        } else if(atomtype == TypeSort.Float64) {
            dvalue = ((Double) result).doubleValue();
        } else
            throw new ForbiddenConversionException();
        return dvalue;
    }

    /**
     * Extract, as an object, n consecutive values
     * of an atomic typed array of values
     *
     * @param dataset  Data containing the objects
     * @param index    Starting element to read
     * @param count    Number of elements to read
     * @return resulting array of values as an object
     */

    /*
    static public Object
    extractVector(DataAtomic dataset, long index, long count, long offset)
        throws DapException
    {
        Object vector = createVector(dataset.getType().getPrimitiveType(),count);
        try {
            dataset.read(index, count, vector, offset);
        } catch (IOException ioe) {
            throw new DapException(ioe);
        }
        return vector;
    }
    */

    /**
     * Convert an array of one type of values to another type
     *
     * @param dsttype target type
     * @param srctype source type
     * @param src     array of values to convert
     * @return resulting array of converted values as an object
     */

    static public Object
    convertVector(DapType dsttype, DapType srctype, Object src)
    {
        int i;

        TypeSort srcatomtype = srctype.getAtomicType();
        TypeSort dstatomtype = dsttype.getAtomicType();

        if(srcatomtype == dstatomtype) {
            return src;
        }
        if(srcatomtype.isIntegerType()
                && TypeSort.getSignedVersion(srcatomtype) == TypeSort.getSignedVersion(dstatomtype))
            return src;

        Object result = CDMTypeFcns.convert(dstatomtype, srcatomtype, src);
        if(result == null)
            throw new ForbiddenConversionException();
        return result;
    }

    /**
     * Given an arbitrary Array (including ArrayStructure), produce
     * a new Array that represents the slice defined by the
     * section.  For now, we create a simple array of the relevant
     * type and fill it by extracting the values specified by the
     * section.
     *

     * param array   the array from which the section is extracted
     * param section determines what to extract
     * throws DapException
     * returns the slice array
     */
    /*static public ucar.ma2.Array
    arraySlice(ucar.ma2.Array array, Section section)
        throws DapException
    {
        // Case it out.
        if(!dapvar.getBaseType().isStructType()) { // =>Atomic type
            if(dapvar.isTopLevel()) {
                // Simplest case: use createview, but watch out for final VLEN
                List<Range> ranges = section.getRanges();
                try {
                    if(CDMUtil.hasVLEN(ranges))
                        return array.section(ranges.subList(0, ranges.size() - 2));
                    else
                        return array.section(ranges);
                } catch (InvalidRangeException ire) {
                    throw new DapException(ire);
                }
            } else
                throw new UnsupportedOperationException(); // same as other cdm
        } else { // struct type
            assert (array instanceof CDMArrayStructure);
            CDMArrayStructure struct = (CDMArrayStructure) array;
            if(dapvar.isTopLevel()) {
                // Build a new ArrayStructure containing
                // the relevant instances.
                int[] shape = section.getShape();
                StructureMembers sm = new StructureMembers(struct.getStructureMembers());
                ArrayStructureMA slice = new ArrayStructureMA(sm, shape);
                CDMOdometer odom = new CDMOdometer(dapvar.getDimensions(), section.getRanges());
                // Compute the number of structuredata instances we need
                long totalsize = section.computeSize();
                List<StructureMembers.Member> mlist = sm.getMembers();
                StructureData[] newdata = new StructureData[(int) totalsize];
                for(int i = 0;odom.hasNext();odom.next(), i++) {
                    long recno = odom.index();
                    StructureDataW clone = new StructureDataW(sm);
                    newdata[i] = clone;
                    StructureData record = struct.getStructureData((int) recno);
                    for(int j = 0;j < mlist.size();j++) {
                        StructureMembers.Member m = mlist.get(j);
                        clone.setMemberData(m, record.getArray(m));
                    }
                }
                return slice;
            } else
                throw new UnsupportedOperationException(); // same as other cdm
        }
    }*/
    static public String
    getChecksumString(byte[] checksum)
    {
        StringBuilder buf = new StringBuilder();
        for(int i = 0; i < checksum.length; i++) {
            byte b = checksum[i];
            buf.append(hexchars.charAt(b >> 4));
            buf.append(hexchars.charAt(b & 0xF));
        }
        return buf.toString();
    }

    /**
     * Convert a Section + variable to a constraint
     *

     *

     * static public View
     * sectionToView(CDMDSP dsp, Variable v, Section section)
     * throws DapException
     * {
     * if(section == null || section.getRank() == 0)
     * return null;
     * // Get the corresponding DapNode
     * DapVariable dv = (DapVariable) dsp.getNode().get(v);
     * if(dv == null)
     * throw new DapException("Variable has no corresponding dap node: " + v.getFullName());
     * // Get the structure path wrt DapDataset for dv
     * // and use path plus the Section to construct a constraint
     * List<DapVariable> structpath = DapUtil.getStructurePath(dv);
     * List<Range> ranges = section.getRanges();
     * View view = new View(dmr);
     * int next = 0;
     * for(int i = 0;i < structpath.size();i++) {
     * dv = structpath.get(i);
     * int rank = dv.getRank();
     * ViewVariable vv = new ViewVariable(dv);
     * List<Slice> slices = new ArrayList<Slice>(rank);
     * for(int j = 0;j < rank;j++, next++) {
     * if(next >= ranges.size())
     * throw new DapException("Range::Rank mismatch");
     * Range range = ranges.get(next);
     * Slice slice = new Slice(range.first(), range.last(), range.stride()).validate();
     * slices.add(slice);
     * }
     * vv.setSlices(slices);
     * view.put(dv, vv);
     * }
     * view.validate(View.EXPAND);
     * return view;
     * }
     */

    static public List<Range>
    dimsetToRanges(List<DapDimension> dimset)
            throws dap4.core.util.DapException
    {
        if(dimset == null)
            return null;
        List<Range> ranges = new ArrayList<>();
        for(int i = 0; i < dimset.size(); i++) {
            DapDimension dim = dimset.get(i);
            try {
                Range r = new Range(dim.getShortName(), 0, (int) dim.getSize() - 1, 1);
                ranges.add(r);
            } catch (InvalidRangeException ire) {
                throw new dap4.core.util.DapException(ire);
            }
        }
        return ranges;
    }

    static public List<Slice>
    shapeToSlices(int[] shape)
            throws dap4.core.util.DapException
    {
        if(shape == null)
            return null;
        List<Slice> slices = new ArrayList<>(shape.length);
        for(int i = 0; i < shape.length; i++) {
            Slice sl = new Slice(0, shape[i] - 1, 1);
            slices.add(sl);
        }
        return slices;
    }

    static public dap4.core.util.Index
    cdmIndexToIndex(ucar.ma2.Index cdmidx)
    {
        int rank = cdmidx.getRank();
        int[] shape = cdmidx.getShape();
        long[] indices = new long[shape.length];
        for(int i = 0; i < rank; i++) {
            indices[i] = shape[i];
        }
        dap4.core.util.Index dapidx = new dap4.core.util.Index(indices, indices);
        return dapidx;
    }

    static public ucar.ma2.Index
    indexToCcMIndex(dap4.core.util.Index d4)
    {
        int rank = d4.getRank();
        int[] shape = new int[rank];
        int[] indices = new int[rank];
        for(int i = 0; i < rank; i++) {
            indices[i] = (int) d4.get(i);
            shape[i] = (int) d4.getSize(i);
        }
        ucar.ma2.Index cdm = ucar.ma2.Index.factory(shape);
        cdm.set(indices);
        return cdm;
    }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.units;

import ucar.unidata.util.Format;
import ucar.units.ConversionException;
import ucar.units.UnitException;

import java.util.StringTokenizer;
import java.util.Date;
import java.util.Calendar;

/**
 * Handles Units that are time durations, eg in seconds, hours, days, years.
 * It keeps track of the original unit name, rather than converting to canonical "seconds".
 * The unit name never changes, but the value may.
 *

 * This is a wrapper around ucar.units.
 * The underlying ucar.units.Unit always has a value of "1.0", ie is a base unit.
 *
 * @author John Caron
 */
public class TimeUnit extends SimpleUnit {
  private double value;
  private double factor = 1.0;
  private String unitString;
  //private Unit uu;

  /**
   * Constructor from a String.
   * @param text [value] <time unit> eg "hours" or "13 hours". Time unit is from udunits.
   * @throws UnitException is bad format
   */
  public TimeUnit(String text) throws UnitException {
    super();

    if (text == null) {
      this.value = 1.0;
      this.unitString = "secs";
      this.uu = SimpleUnit.makeUnit( unitString); // always a base unit
      return;
    }

    StringTokenizer stoker = new StringTokenizer(text);
    int ntoke = stoker.countTokens();
    if (ntoke == 1) {
      this.value = 1.0;
      this.unitString = stoker.nextToken();

    } else if (ntoke == 2){
      this.value = Double.parseDouble(stoker.nextToken());
      this.unitString = stoker.nextToken();
    } else
      throw new IllegalArgumentException("Not TimeUnit = "+text);

    uu = SimpleUnit.makeUnit( unitString); // always a base unit
    factor = uu.convertTo( 1.0, SimpleUnit.secsUnit);
  }

  /**
   * Constructor from a value and a unit name.
   * @param value amount of the unit.
   * @param unitString  Time unit string from udunits.
   * @throws UnitException if parse fails
   */
  public TimeUnit(double value, String unitString) throws UnitException {
    this.value = value;
    this.unitString = unitString;
    uu = SimpleUnit.makeUnit( unitString);
    factor = uu.convertTo( 1.0, SimpleUnit.secsUnit);
  }

  /**
   * Copy Constructor.
   * @param src copy from here
   */
  public TimeUnit( TimeUnit src) {
    this.value = src.getValue();
    this.unitString = src.getUnitString();
    uu = src.uu;
    factor = src.getFactor();
  }

  /** Get the value. */
  @Override
  public double getValue() { return value; }

  /** Get the factor that converts this unit to seconds.
   * getValueInSeconds = factor * value
   * @return factor that converts this unit to seconds.
   */
  public double getFactor() { return factor; }

  /** Set the value in the original units.
   * @param value set value, must be in units of this
   */
  public void setValue( double value) {
    this.value = value;
  }

  /** Get the "base" unit String, eg "secs" or "days" */
  @Override
  public String getUnitString() { return unitString; }

  /** String representation. */
  @Override
  public String toString() {
    return Format.d(value, 5) + " "+unitString;
  }

  /** Get the time duration in seconds.
   * @return get current value in units of seconds
   */
  public double getValueInSeconds() {
    return factor * value;
  }

  /** Get the time duration in seconds of the specified value.
   * @param value convert this value, must be in units of this
   * @return get the value in units of seconds
   */
  public double getValueInSeconds(double value) {
    return factor * value;
  }

  /**
   * Set the value, using the given number of seconds.
   * @param secs : number of seconds; convert this to the units of this TimeUnit.
   */
  public void setValueInSeconds( double secs) {
    value = secs/factor;
  }

  // override

  /** Convert given value of this unit to the new unit.
   *  NOTE: the current value of this unit ignored, the given value is used instead.
   * This is different than ucar.units or SimpleUnit.
   * @param value in the units of this "base unit"
   * @param outputUnit convert to this base type, must be convertible to units of "seconds"
   * @return new value in the units of the "outputUnit
   * @throws ConversionException
   */
  public double convertTo(double value, TimeUnit outputUnit) throws ConversionException {
    return uu.convertTo( value, outputUnit.uu);
  }

  /**
   * Add the time amount to the given Date, return a new Date.
   * @param d add to this Date
   * @return Date with getValueInSeconds() added to it.
   */
  public Date add( Date d) {
    Calendar cal = Calendar.getInstance();
    cal.setTime( d);
    cal.add( Calendar.SECOND, (int) getValueInSeconds());
    return cal.getTime();
  }

  /** TimeUnits with same value and unitString are equal */
  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if (!(o instanceof TimeUnit)) return false;
    return o.hashCode() == this.hashCode();
  }

  /** Override hashcode to be consistent with equals. */
  @Override
  public int hashCode() {
    if (hashCode == 0) {
      int result = 17;
      result = 37*result + unitString.hashCode();
      result = 37*result + (int) (1000.0 * value);
      hashCode = result;
    }
    return hashCode;
  }
  private int hashCode = 0;

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.units;

import javax.annotation.concurrent.Immutable;

/**
 * Provides support for a unit that is a mutiplicative factor of a reference
 * unit.
 *
 * Instances of this class are immutable.
 *
 * @author Steven R. Emmerson
 */
@Immutable
public final class ScaledUnit extends UnitImpl implements DerivableUnit {
    private static final long serialVersionUID = 1L;

    /**
     * The multiplicative factor.
     *
     * @serial
     */
    private final double      _scale;

    /**
     * The reference unit.
     *
     * @serial
     */
    private final Unit        _unit;

    /**
     * Constructs from a multiplicative factor. Returns a dimensionless unit
     * whose value is the multiplicative factor rather than unity.
     *
     * @param scale
     *            The multiplicative factor.
     */
    public ScaledUnit(final double scale) {
        this(scale, DerivedUnitImpl.DIMENSIONLESS);
    }

    /**
     * Constructs from a multiplicative factor and a reference unit.
     *
     * @param scale
     *            The multiplicative factor.
     * @param unit
     *            The reference unit.
     */
    public ScaledUnit(final double scale, final Unit unit) {
        this(scale, unit, null);
    }

    /**
     * Constructs from a multiplicative factor, a reference unit, and an
     * identifier.
     *
     * @param scale
     *            The multiplicative factor.
     * @param unit
     *            The reference unit.
     * @param id
     *            The identifier for the unit.
     */
    public ScaledUnit(final double scale, final Unit unit, final UnitName id) {
        super(id);
        if (!(unit instanceof ScaledUnit)) {
            _unit = unit;
            _scale = scale;
        }
        else {
            _unit = ((ScaledUnit) unit)._unit;
            _scale = ((ScaledUnit) unit)._scale * scale;
        }
    }

    static Unit getInstance(final double scale, final Unit unit)
            throws MultiplyException {
        if (scale == 0) {
            throw new MultiplyException(scale, unit);
        }
        return scale == 1
                ? unit
                : new ScaledUnit(scale, unit);
    }

    /**
     * Returns the multiplicative factor.
     *
     * @return The multiplicative factor.
     */
    public double getScale() {
        return _scale;
    }

    /**
     * Returns the reference unit.
     *
     * @return The reference unit.
     */
    public Unit getUnit() {
        return _unit;
    }

    /*
     * From UnitImpl:
     */

    /**
     * Clones this unit, changing the identifier.
     *
     * @param id
     *            The new identifier.
     * @return A ScaledUnit with the new identifier.
     */
    public Unit clone(final UnitName id) {
        return new ScaledUnit(_scale, getUnit(), id);
    }

    @Override
    public Unit multiplyBy(final double scale) throws MultiplyException {
        return getInstance(scale * _scale, _unit);
    }

    /**
     * Multiplies this unit by another unit.
     *
     * @param that
     *            The other unit.
     * @return The product of this unit and the other unit.
     * @throws MultiplyException
     *             Can't multiply these units together.
     */
    @Override
    protected Unit myMultiplyBy(final Unit that) throws MultiplyException {
        return that instanceof ScaledUnit
                ? new ScaledUnit(getScale() * ((ScaledUnit) that).getScale(),
                        getUnit().multiplyBy(((ScaledUnit) that).getUnit()))
                : new ScaledUnit(getScale(), getUnit().multiplyBy(that));
    }

    /**
     * Divides this unit by another unit.
     *
     * @param that
     *            The other unit.
     * @return The quotient of this unit divided by the other unit.
     * @throws OperationException
     *             Can't divide these units.
     */
    @Override
    protected Unit myDivideBy(final Unit that) throws OperationException {
        return that instanceof ScaledUnit
                ? new ScaledUnit(getScale() / ((ScaledUnit) that).getScale(),
                        getUnit().divideBy(((ScaledUnit) that).getUnit()))
                : new ScaledUnit(getScale(), getUnit().divideBy(that));
    }

    /**
     * Divides this unit into another unit.
     *
     * @param that
     *            The other unit.
     * @return The quotient of this unit divided into the other unit.
     * @throws OperationException
     *             Can't divide these units.
     */
    @Override
    protected Unit myDivideInto(final Unit that) throws OperationException {
        return that instanceof ScaledUnit
                ? new ScaledUnit(((ScaledUnit) that).getScale() / getScale(),
                        getUnit().divideInto(((ScaledUnit) that).getUnit()))
                : new ScaledUnit(1 / getScale(), getUnit().divideInto(that));
    }

    /**
     * Raises this unit to a power.
     *
     * @param power
     *            The power.
     * @return The result of raising this unit to the power.
     * @throws RaiseException
     *             Can't raise this unit to a power.
     */
    @Override
    protected Unit myRaiseTo(final int power) throws RaiseException {
        return new ScaledUnit(Math.pow(getScale(), power), getUnit().raiseTo(
                power));
    }

    /**
     * Gets the derived unit underlying this unit.
     *
     * @return The derived unit which underlies this unit.
     */
    public DerivedUnit getDerivedUnit() {
        return getUnit().getDerivedUnit();
    }

    /**
     * Converts a numeric value from this unit to the underlying derived unit.
     *
     * @param amount
     *            The numeric value in this unit.
     * @return The equivalent value in the underlying derived unit.
     * @throws ConversionException
     *             Can't convert value to the underlying derived unit.
     */
    public float toDerivedUnit(final float amount) throws ConversionException {
        return (float) toDerivedUnit((double) amount);
    }

    /**
     * Converts a numeric value from this unit to the underlying derived unit.
     *
     * @param amount
     *            The numeric value in this unit.
     * @return The equivalent value in the underlying derived unit.
     * @throws ConversionException
     *             Can't convert value to the underlying derived unit.
     */
    public double toDerivedUnit(final double amount) throws ConversionException {
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(this, getDerivedUnit());
        }
        return ((DerivableUnit) _unit).toDerivedUnit(amount * getScale());
    }

    /**
     * Converts numeric values from this unit to the underlying derived unit.
     *
     * @param input
     *            The numeric values in this unit.
     * @param output
     *            The equivalent values in the underlying derived unit.
     * @return output.
     * @throws ConversionException
     *             Can't convert values to the underlying derived unit.
     */
    public float[] toDerivedUnit(final float[] input, final float[] output)
            throws ConversionException {
        final float scale = (float) getScale();
        for (int i = input.length; --i >= 0;) {
            output[i] = input[i] * scale;
        }
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(this, getDerivedUnit());
        }
        return ((DerivableUnit) getUnit()).toDerivedUnit(output, output);
    }

    /**
     * Converts numeric values from this unit to the underlying derived unit.
     *
     * @param input
     *            The numeric values in this unit.
     * @param output
     *            The equivalent values in the underlying derived unit.
     * @return output.
     * @throws ConversionException
     *             Can't convert values to the underlying derived unit.
     */
    public double[] toDerivedUnit(final double[] input, final double[] output)
            throws ConversionException {
        final double scale = getScale();
        for (int i = input.length; --i >= 0;) {
            output[i] = input[i] * scale;
        }
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(this, getDerivedUnit());
        }
        return ((DerivableUnit) getUnit()).toDerivedUnit(output, output);
    }

    /**
     * Converts a numeric value from the underlying derived unit to this unit.
     *
     * @param amount
     *            The numeric value in the underlying derived unit.
     * @return The equivalent value in this unit.
     * @throws ConversionException
     *             Can't convert value.
     */
    public float fromDerivedUnit(final float amount) throws ConversionException {
        return (float) fromDerivedUnit((double) amount);
    }

    /**
     * Converts a numeric value from the underlying derived unit to this unit.
     *
     * @param amount
     *            The numeric value in the underlying derived unit.
     * @return The equivalent value in this unit.
     * @throws ConversionException
     *             Can't convert value.
     */
    public double fromDerivedUnit(final double amount)
            throws ConversionException {
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(getDerivedUnit(), this);
        }
        return ((DerivableUnit) getUnit()).fromDerivedUnit(amount) / getScale();
    }

    /**
     * Converts numeric values from the underlying derived unit to this unit.
     *
     * @param input
     *            The numeric values in the underlying derived unit.
     * @param output
     *            The equivalent values in this unit.
     * @return output.
     * @throws ConversionException
     *             Can't convert values.
     */
    public float[] fromDerivedUnit(final float[] input, final float[] output)
            throws ConversionException {
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(getDerivedUnit(), this);
        }
        ((DerivableUnit) getUnit()).fromDerivedUnit(input, output);
        final float scale = (float) getScale();
        for (int i = input.length; --i >= 0;) {
            output[i] /= scale;
        }
        return output;
    }

    /**
     * Converts numeric values from the underlying derived unit to this unit.
     *
     * @param input
     *            The numeric values in the underlying derived unit.
     * @param output
     *            The equivalent values in this unit.
     * @return output.
     * @throws ConversionException
     *             Can't convert values.
     */
    public double[] fromDerivedUnit(final double[] input, final double[] output)
            throws ConversionException {
        if (!(_unit instanceof DerivableUnit)) {
            throw new ConversionException(getDerivedUnit(), this);
        }
        ((DerivableUnit) getUnit()).fromDerivedUnit(input, output);
        final double scale = getScale();
        for (int i = input.length; --i >= 0;) {
            output[i] /= scale;
        }
        return output;
    }

    /**
     * Indicates if this unit is semantically identical to an object.
     *
     * @param object
     *            The object.
     * @return true if an only if this unit is semantically
     *         identical to object
     *				.
     */
    public boolean equals(final Object object) {
        if (this == object) {
            return true;
        }
        if (_scale == 1) {
            return object.equals(_unit);
        }
        if (!(object instanceof ScaledUnit)) {
            return false;
        }
        ScaledUnit that = (ScaledUnit) object;
        if (Double.compare(that._scale, _scale) != 0) return false;
        if (!_unit.equals(that._unit)) return false;
        return true;
    }

  /**
     * Returns the hash code of this instance.
     *
     * @return The hash code of this instance.
     */
    @Override
    public int hashCode() {
        return (getScale() == 1
                ? 0
                : Double.valueOf(getScale()).hashCode()) ^ getUnit().hashCode();
    }

    /**
     * Indicates if this unit is dimensionless. A ScaledUnit is dimensionless if
     * and only if the reference unit is dimensionless.
     *
     * @return true if and only if this unit is dimensionless.
     */
    public boolean isDimensionless() {
        return getUnit().isDimensionless();
    }

    /**
     * Returns the string representation of this unit.
     *
     * @return The string representation of this unit.
     */
    @Override
    public String toString() {
        final String string = super.toString(); // get symbol or name
        return string != null
                ? string
                : getCanonicalString();
    }

    /**
     * Returns the canonical string representation of the unit.
     *
     * @return The canonical string representation.
     */
    public String getCanonicalString() {
        return DerivedUnitImpl.DIMENSIONLESS.equals(_unit)
                ? Double.toString(getScale())
                : Double.toString(getScale()) + " " + _unit.toString();
    }

    /**
     * Tests this class.
     */
    public static void main(final String[] args) throws Exception {
        final BaseUnit meter = BaseUnit.getOrCreate(UnitName.newUnitName(
                "meter", null, "m"), BaseQuantity.LENGTH);
        final ScaledUnit nauticalMile = new ScaledUnit(1852f, meter);
        System.out.println("nauticalMile.getUnit().equals(meter)="
                + nauticalMile.getUnit().equals(meter));
        final ScaledUnit nauticalMileMeter = (ScaledUnit) nauticalMile
                .multiplyBy(meter);
        System.out.println("nauticalMileMeter.divideBy(nauticalMile)="
                + nauticalMileMeter.divideBy(nauticalMile));
        System.out.println("meter.divideBy(nauticalMile)="
                + meter.divideBy(nauticalMile));
        System.out
                .println("nauticalMile.raiseTo(2)=" + nauticalMile.raiseTo(2));
        System.out.println("nauticalMile.toDerivedUnit(1.)="
                + nauticalMile.toDerivedUnit(1.));
        System.out
                .println("nauticalMile.toDerivedUnit(new float[]{1,2,3}, new float[3])[1]="
                        + nauticalMile.toDerivedUnit(new float[] { 1, 2, 3 },
                                new float[3])[1]);
        System.out.println("nauticalMile.fromDerivedUnit(1852.)="
                + nauticalMile.fromDerivedUnit(1852.));
        System.out
                .println("nauticalMile.fromDerivedUnit(new float[]{1852},new float[1])[0]="
                        + nauticalMile.fromDerivedUnit(new float[] { 1852 },
                                new float[1])[0]);
        System.out.println("nauticalMile.equals(nauticalMile)="
                + nauticalMile.equals(nauticalMile));
        final ScaledUnit nautical2Mile = new ScaledUnit(2, nauticalMile);
        System.out.println("nauticalMile.equals(nautical2Mile)="
                + nauticalMile.equals(nautical2Mile));
        System.out.println("nauticalMile.isDimensionless()="
                + nauticalMile.isDimensionless());
        final BaseUnit radian = BaseUnit.getOrCreate(UnitName.newUnitName(
                "radian", null, "rad"), BaseQuantity.PLANE_ANGLE);
        final ScaledUnit degree = new ScaledUnit(3.14159 / 180, radian);
        System.out.println("degree.isDimensionless()="
                + degree.isDimensionless());
    }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import com.google.common.base.Throwables;
import javax.annotation.Nonnull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.nc2.grib.coord.CoordinateTime2D;
import ucar.ma2.*;
import ucar.nc2.ft2.coverage.CoordsSet;
import ucar.nc2.ft2.coverage.SubsetParams;
import ucar.nc2.grib.GdsHorizCoordSys;
import ucar.nc2.grib.coord.TimeCoordIntvDateValue;
import ucar.nc2.grib.grib1.Grib1ParamTime;
import ucar.nc2.grib.grib1.Grib1Parameter;
import ucar.nc2.grib.grib1.Grib1Record;
import ucar.nc2.grib.grib1.Grib1SectionProductDefinition;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib2.Grib2Record;
import ucar.nc2.grib.grib2.Grib2RecordScanner;
import ucar.nc2.grib.grib2.table.Grib2Tables;
import ucar.nc2.util.Misc;
import ucar.unidata.io.RandomAccessFile;

import javax.annotation.concurrent.Immutable;
import java.io.IOException;
import java.util.*;

/**
 * Grib Data Reader.
 * Split from GribIosp, so can be used by GribCoverage.
 *
 * @author caron
 * @since 4/6/11
 */
@Immutable
public abstract class GribDataReader {
  private static final Logger logger = LoggerFactory.getLogger(GribDataReader.class);

  public static GribDataReader factory(GribCollectionImmutable gribCollection, GribCollectionImmutable.VariableIndex vindex) {
    if (gribCollection.isGrib1)
      return new Grib1DataReader(gribCollection, vindex);
    else
      return new Grib2DataReader(gribCollection, vindex);
  }

  protected abstract float[] readData(RandomAccessFile rafData, DataRecord dr) throws IOException;
  protected abstract void show(RandomAccessFile rafData, long dataPos) throws IOException;

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
  public static GribCollectionImmutable.Record currentDataRecord;
  public static GribDataValidator validator;
  public static String currentDataRafFilename;
  static boolean show = false;   // debug

  protected final GribCollectionImmutable gribCollection;
  private final GribCollectionImmutable.VariableIndex vindex;
  private final List<DataRecord> records = new ArrayList<>();

  protected GribDataReader(GribCollectionImmutable gribCollection, GribCollectionImmutable.VariableIndex vindex) {
    this.gribCollection = gribCollection;
    this.vindex = vindex;
  }

  /**
   * Read the section of data described by want
   * @param want which data do you want?
   * @return data as an Array
   */
  public Array readData(SectionIterable want) throws IOException, InvalidRangeException {
    if (vindex instanceof PartitionCollectionImmutable.VariableIndexPartitioned)
      return readDataFromPartition((PartitionCollectionImmutable.VariableIndexPartitioned) vindex, want);
    else
      return readDataFromCollection(vindex, want);
  }

  /*
   SectionIterable iterates over the source indexes, corresponding to vindex's SparseArray.
     IOSP: works because variable coordinate corresponds 1-1 to Grib Coordinate.
     GribCoverage: must translate coordinates to Grib Coordinate index.
   want.getShape() indicates the result Array shape.
   SectionIterable.next(int[] index) is not used here.
   */
  private Array readDataFromCollection(GribCollectionImmutable.VariableIndex vindex, SectionIterable want) throws IOException {
    // first time, read records and keep in memory
    vindex.readRecords();

    int rank = want.getRank();
    int sectionLen = rank - 2; // all but x, y
    SectionIterable sectionWanted = want.subSection(0, sectionLen);
    // assert sectionLen == vindex.getRank(); LOOK true or false ??

    // collect all the records that need to be read
    int resultIndex = 0;
    for  (int sourceIndex : sectionWanted) {
      // addRecord(sourceIndex, count++);
      GribCollectionImmutable.Record record = vindex.getRecordAt(sourceIndex);
      if (Grib.debugRead)
        logger.debug("GribIosp debugRead sourceIndex=%d resultIndex=%d record is null=%s%n", sourceIndex, resultIndex, record == null);
      if (record != null)
        records.add( new DataRecord(resultIndex, record, vindex.group.getGdsHorizCoordSys()));
      resultIndex++;
    }

    // sort by file and position, then read
    DataReceiverIF dataReceiver = new DataReceiver(want.getShape(), want.getRange(rank - 2), want.getRange(rank-1) );
    read(dataReceiver);
    return dataReceiver.getArray();
  }

  /*
   Iterates using SectionIterable.next(int[] index).
   The work of translating that down into partition heirarchy and finally to a GC is all in VariableIndexPartitioned.getDataRecord(int[] index)
   want.getShape() indicates the result Array shape.
 */
  private Array readDataFromPartition(PartitionCollectionImmutable.VariableIndexPartitioned vindexP, SectionIterable section) throws IOException {

    int rank = section.getRank();
    SectionIterable sectionWanted = section.subSection(0, rank - 2); // all but x, y
    SectionIterable.SectionIterator iterWanted = sectionWanted.getIterator();  // iterator over wanted indices in vindexP
    int[] indexWanted = new int[rank - 2];                              // place to put the iterator result
    int[] useIndex = indexWanted;

    // collect all the records that need to be read
    int resultPos = 0;
    while (iterWanted.hasNext()) {
      iterWanted.next(indexWanted);   // returns the vindexP index in indexWanted array

      // for MRUTP, must munge the index here (not in vindexP.getDataRecord, because its recursive
      if (vindexP.getType() == GribCollectionImmutable.Type.MRUTP) {
        // find the partition from getRuntimeIdxFromMrutpTimeIndex
        CoordinateTime2D time2D = (CoordinateTime2D) vindexP.getCoordinateTime();
        assert time2D != null;
        int[] timeIndices = time2D.getTimeIndicesFromMrutp(indexWanted[0]);

        int[] indexReallyWanted = new int[indexWanted.length+1];
        indexReallyWanted[0] = timeIndices[0];
        indexReallyWanted[1] = timeIndices[1];
        System.arraycopy(indexWanted, 1, indexReallyWanted, 2, indexWanted.length-1);
        useIndex = indexReallyWanted;
      }

      PartitionCollectionImmutable.DataRecord record = vindexP.getDataRecord(useIndex);
      if (record == null) {
        if (Grib.debugRead) logger.debug("readDataFromPartition missing data%n");
        resultPos++; // can just skip, since result is prefilled with NaNs
        continue;
      }
      record.resultIndex = resultPos;
      records.add(record);
      resultPos++;
    }

    // sort by file and position, then read
    DataReceiverIF dataReceiver = new DataReceiver(section.getShape(), section.getRange(rank-2), section.getRange(rank-1) );
    readPartitioned(dataReceiver);

    return dataReceiver.getArray();
  }

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
  // Coordinate based subsetting for Coverage

  public Array readData2(CoordsSet want, RangeIterator yRange, RangeIterator xRange) throws IOException {
    if (vindex instanceof PartitionCollectionImmutable.VariableIndexPartitioned)
      return readDataFromPartition2((PartitionCollectionImmutable.VariableIndexPartitioned) vindex, want, yRange, xRange);
    else
      return readDataFromCollection2(vindex, want, yRange, xRange);
  }

  private Array readDataFromCollection2(GribCollectionImmutable.VariableIndex vindex, CoordsSet want, RangeIterator yRange, RangeIterator xRange) throws IOException {
    // first time, read records and keep in memory
    vindex.readRecords();

    // collect all the records that need to be read
    int resultIndex = 0;
    for (SubsetParams coords : want) {
      GribCollectionImmutable.Record record = vindex.getRecordAt(coords);
      if (record != null) {
        DataRecord dr = new DataRecord(resultIndex, record, vindex.group.getGdsHorizCoordSys());
        if (GribDataReader.validator != null) dr.validation = coords;
        records.add(dr);
      }
      resultIndex++;
    }

    DataReceiverIF dataReceiver = new DataReceiver(want.getShape(yRange, xRange), yRange, xRange );
    read(dataReceiver);
    return dataReceiver.getArray();
  }

  private Array readDataFromPartition2(PartitionCollectionImmutable.VariableIndexPartitioned vindexP, CoordsSet want, RangeIterator yRange, RangeIterator xRange) throws IOException {

    // collect all the records that need to be read
    int resultPos = 0;
    for (SubsetParams coords : want) {
      PartitionCollectionImmutable.DataRecord record = vindexP.getDataRecord(coords);
      if (record != null) {
        record.resultIndex = resultPos;
        records.add(record);
      }
      resultPos++;
    }

    // sort by file and position, then read
    DataReceiverIF dataReceiver = new DataReceiver(want.getShape(yRange, xRange), yRange, xRange );
    readPartitioned(dataReceiver);

    return dataReceiver.getArray();
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Read all of the data records that have been added.
   * The full (x,y) record is read, the reciever will subset the (x, y) as needed.
   * @param dataReceiver send data here.
   */
  private void read(DataReceiverIF dataReceiver) throws IOException {
    Collections.sort(records);

    int currFile = -1;
    RandomAccessFile rafData = null;
    try {
      for (DataRecord dr : records) {
        if (Grib.debugIndexOnly || Grib.debugGbxIndexOnly) {
          GribIosp.debugIndexOnlyCount++;
          currentDataRecord = dr.record;
          currentDataRafFilename = gribCollection.getDataRafFilename(dr.record.fileno);
          if (Grib.debugIndexOnlyShow) dr.show(gribCollection);
          dataReceiver.setDataToZero();
          continue;
        }

        if (dr.record.fileno != currFile) {
          if (rafData != null) rafData.close();
          rafData = gribCollection.getDataRaf(dr.record.fileno);
          currFile = dr.record.fileno;
        }

        if (dr.record.pos == GribCollectionMutable.MISSING_RECORD) continue;

        if (GribDataReader.validator != null && dr.validation != null && rafData != null) {
          GribDataReader.validator.validate(gribCollection.cust, rafData, dr.record.pos + dr.record.drsOffset, dr.validation);

        } else if (show && rafData != null) { // for validation
          show( dr.validation);
          show(rafData, dr.record.pos + dr.record.drsOffset);
        }

        float[] data = readData(rafData, dr);
        GdsHorizCoordSys hcs = vindex.group.getGdsHorizCoordSys();
        dataReceiver.addData(data, dr.resultIndex, hcs.nx);
      }

    } finally {
      if (rafData != null) rafData.close();  // make sure its closed even on exception
    }
  }

  private void show(SubsetParams validation) {
    if (validation == null) return;
    System.out.printf("Coords wanted%n %s", validation);
  }

  private void readPartitioned(DataReceiverIF dataReceiver) throws IOException {
    Collections.sort(records);

    PartitionCollectionImmutable.DataRecord lastRecord = null;
    RandomAccessFile rafData = null;
    try {

      for (DataRecord dr : records) {
        PartitionCollectionImmutable.DataRecord drp = (PartitionCollectionImmutable.DataRecord) dr;
        if (Grib.debugIndexOnly || Grib.debugGbxIndexOnly) {
          GribIosp.debugIndexOnlyCount++;
          if (Grib.debugIndexOnlyShow) drp.show();
          dataReceiver.setDataToZero();
          continue;
        }

        if ((rafData == null) || !drp.usesSameFile(lastRecord)) {
          if (rafData != null) rafData.close();
          rafData = drp.usePartition.getRaf(drp.partno, dr.record.fileno);
        }
        lastRecord = drp;

        if (dr.record.pos == GribCollectionMutable.MISSING_RECORD) continue;

        if (GribDataReader.validator != null && dr.validation != null) {
          GribDataReader.validator.validate(gribCollection.cust, rafData, dr.record.pos + dr.record.drsOffset, dr.validation);
        } else if (show) { // for validation
          show( dr.validation);
          show(rafData, dr.record.pos + dr.record.drsOffset);
        }

        float[] data = readData(rafData, dr);
        GdsHorizCoordSys hcs = dr.hcs;
        dataReceiver.addData(data, dr.resultIndex, hcs.nx);
      }

    } finally {
      if (rafData != null) rafData.close();  // make sure its closed even on exception
    }
  }

  public static class DataRecord implements Comparable<DataRecord> {
    int resultIndex; // index into the result array
    final GribCollectionImmutable.Record record;
    final GdsHorizCoordSys hcs;
    SubsetParams validation;

    DataRecord(int resultIndex, GribCollectionImmutable.Record record, GdsHorizCoordSys hcs) {
      this.resultIndex = resultIndex;
      this.record = record;
      this.hcs = hcs;
    }

    @Override
    public int compareTo(@Nonnull DataRecord o) {
      int r = Misc.compare(record.fileno, o.record.fileno);
      if (r != 0) return r;
      return Misc.compare(record.pos, o.record.pos);
    }

    // debugging
    public void show(GribCollectionImmutable gribCollection) {
      String dataFilename = gribCollection.getFilename(record.fileno);
      System.out.printf(" fileno=%d filename=%s startPos=%d%n", record.fileno, dataFilename, record.pos);
    }
  }

  public interface DataReceiverIF {
    void addData(float[] data, int resultIndex, int nx);
    void setDataToZero(); // only used when debugging with gbx/ncx only, to fake the data
    Array getArray();
  }

  public static class DataReceiver implements DataReceiverIF {
    private Array dataArray;
    private final RangeIterator yRange;
    private final RangeIterator xRange;
    private final int horizSize;

    DataReceiver(int[] shape, RangeIterator yRange, RangeIterator xRange) {
      this.yRange = yRange;
      this.xRange = xRange;
      this.horizSize = yRange.length() * xRange.length();

      long len = Section.computeSize(shape);
      if (len > 100 * 1000 * 1000*4) { // LOOK make configurable
        logger.debug("Len greater that 100MB shape={}%n{}", Misc.showInts(shape),
                Throwables.getStackTraceAsString(new Throwable()));
        throw new IllegalArgumentException("RequestTooLarge: Len greater that 100M ");
      }
      float[] data = new float[ (int) len];
      Arrays.fill(data, Float.NaN); // prefill primitive array
      dataArray = Array.factory(DataType.FLOAT, shape, data);
    }

    @Override
    public void addData(float[] data, int resultIndex, int nx) {
      int start = resultIndex * horizSize;
      int count = 0;
      for (int y : yRange) {
        for (int x : xRange) {
          int dataIdx = y * nx + x;
          dataArray.setFloat(start + count, data[dataIdx]);
          count++;
        }
      }
    }

    // optimization
    @Override
    public void setDataToZero() {
      float[] data = (float[]) dataArray.get1DJavaArray(dataArray.getDataType());
      Arrays.fill(data, 0.0f);
    }

    @Override
    public Array getArray() {
      return dataArray;
    }
  }

    /* public static class ChannelReceiver implements DataReceiverIF {
    private WritableByteChannel channel;
    private DataOutputStream outStream;
    private Range yRange, xRange;

    ChannelReceiver(WritableByteChannel channel, Range yRange, Range xRange) {
      this.channel = channel;
      this.outStream = new DataOutputStream(Channels.newOutputStream(channel));
      this.yRange = yRange;
      this.xRange = xRange;
    }

    @Override
    public void addData(float[] data, int resultIndex, int nx) throws IOException {
      // LOOK: write some ncstream header
      // outStream.write(header);

      // now write the data
      for (int y = yRange.first(); y <= yRange.last(); y += yRange.stride()) {
        for (int x = xRange.first(); x <= xRange.last(); x += xRange.stride()) {
          int dataIdx = y * nx + x;
          outStream.writeFloat(data[dataIdx]);
        }
      }
    }

    // optimization
    @Override
    public void setDataToZero() {
    }

    @Override
    public Array getArray() {
      return null;
    }
  }    */

  /////////////////////////////////////////////////////////

  private static class Grib2DataReader extends GribDataReader {
    private final Grib2Tables cust;

    Grib2DataReader(GribCollectionImmutable gribCollection,
        GribCollectionImmutable.VariableIndex vindex) {
      super(gribCollection, vindex);
      this.cust = (Grib2Tables) gribCollection.cust;
    }

    @Override
    protected float[] readData(RandomAccessFile rafData, GribDataReader.DataRecord dr) throws IOException {
      GdsHorizCoordSys hcs = dr.hcs;
      long dataPos = dr.record.pos + dr.record.drsOffset;
      long bmsPos = (dr.record.bmsOffset > 0) ? dr.record.pos + dr.record.bmsOffset : 0;
      return Grib2Record.readData(rafData, dataPos, bmsPos, hcs.gdsNumberPoints, hcs.getScanMode(),
              hcs.nxRaw, hcs.nyRaw, hcs.nptsInLine);
    }

    @Override
    protected void show(RandomAccessFile rafData, long pos) throws IOException {
      Grib2Record gr = Grib2RecordScanner.findRecordByDrspos(rafData, pos);
      if (gr != null) {
        Formatter f = new Formatter();
        f.format("File=%s%n", rafData.getLocation());
        f.format("  Parameter=%s%n", cust.getVariableName(gr));
        f.format("  ReferenceDate=%s%n", gr.getReferenceDate());
        f.format("  ForecastDate=%s%n", cust.getForecastDate(gr));
        TimeCoordIntvDateValue tinv = cust.getForecastTimeInterval(gr);
        if (tinv != null) f.format("  TimeInterval=%s%n", tinv);
        f.format("  ");
        gr.getPDS().show(f);
        System.out.printf("%nGrib2Record.readData at drsPos %d = %s%n", pos, f.toString());
      }
    }
  }

  private static class Grib1DataReader extends GribDataReader {
    private final Grib1Customizer cust;

    Grib1DataReader(GribCollectionImmutable gribCollection,
        GribCollectionImmutable.VariableIndex vindex) {
      super(gribCollection, vindex);
      this.cust = (Grib1Customizer) gribCollection.cust;
    }

    @Override
    protected float[] readData(RandomAccessFile rafData, DataRecord dr) throws IOException {
      return Grib1Record.readData(rafData, dr.record.pos);
    }

    @Override
    protected void show(RandomAccessFile rafData, long dataPos) throws IOException {
      rafData.seek(dataPos);
      Grib1Record gr = new Grib1Record(rafData);
      Formatter f = new Formatter();
      f.format("File=%s%n", rafData.getLocation());
      Grib1SectionProductDefinition pds = gr.getPDSsection();
      Grib1Parameter param = cust.getParameter(pds.getCenter(), pds.getSubCenter(), pds.getTableVersion(), pds.getParameterNumber());
      f.format("  Parameter=%s%n", param);
      f.format("  ReferenceDate=%s%n", gr.getReferenceDate());
      Grib1ParamTime ptime = gr.getParamTime(cust);
      f.format("  ForecastTime=%d%n", ptime.getForecastTime());
      if (ptime.isInterval()) {
        int[] tinv = ptime.getInterval();
        f.format("  TimeInterval=(%d,%d)%n", tinv[0], tinv[1]);
      }
      f.format("%n");
      gr.getPDSsection().showPds(cust, f);
      System.out.printf("%nGrib1Record.readData at drsPos %d = %s%n", dataPos, f.toString());
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ft.point;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import ucar.nc2.VariableSimpleIF;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.ft.PointFeature;
import ucar.nc2.ft.PointFeatureCollection;
import ucar.nc2.ft.PointFeatureCollectionIterator;
import ucar.nc2.ft.StationTimeSeriesFeature;
import ucar.nc2.ft.StationTimeSeriesFeatureCollection;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.time.CalendarDateUnit;
import ucar.nc2.util.IOIterator;
import ucar.unidata.geoloc.LatLonRect;

/**
 * Abstract superclass for implementations of StationFeatureCollection.
 * Subclass must supply createStationHelper, may need to override getPointFeatureCollectionIterator().
 *
 * @author caron
 * @since Feb 5, 2008
 */
public abstract class StationTimeSeriesCollectionImpl extends PointFeatureCCImpl implements StationTimeSeriesFeatureCollection {
  private volatile StationHelper stationHelper;

  public StationTimeSeriesCollectionImpl(String name, CalendarDateUnit timeUnit, String altUnits) {
    super(name, timeUnit, altUnits, FeatureType.STATION);
  }

  // Double-check idiom for lazy initialization of instance fields. See Effective Java 2nd Ed, p. 283.
  protected StationHelper getStationHelper() {
    if (stationHelper == null) {
      synchronized (this) {
        if (stationHelper == null) {
          try {
            stationHelper = createStationHelper();
          } catch (IOException e) {
            // The methods that will call getStationHelper() aren't declared to throw IOException, so we must
            // wrap it in an unchecked exception.
            throw new RuntimeException(e);
          }
        }
      }
    }

    assert stationHelper != null : "stationHelper is null for StationTimeSeriesCollectionImpl"+getName();
    return stationHelper;
  }

  // Allow StationHelper to be lazily initialized.
  protected abstract StationHelper createStationHelper() throws IOException;

  @Override
  public LatLonRect getBoundingBox() {
    return getStationHelper().getBoundingBox();
  }

  @Override
  public List<StationFeature> getStationFeatures() throws IOException {
    return getStationHelper().getStationFeatures();
  }

  @Override
  public List<StationFeature> getStationFeatures(List<String> stnNames) {
    return getStationHelper().getStationFeaturesFromNames(stnNames);
  }

  @Override
  public List<StationFeature> getStationFeatures(ucar.unidata.geoloc.LatLonRect boundingBox) throws IOException {
    return getStationHelper().getStationFeatures(boundingBox);
  }

  @Override
  public StationFeature findStationFeature(String name) {
    return getStationHelper().getStation(name);
  }

  @Override
  public StationTimeSeriesFeature getStationTimeSeriesFeature(StationFeature s) {
    return (StationTimeSeriesFeature) s; // LOOK
  }

  ////////////////////////////////////////////////////////////////
  // subset

  @Override
  public StationTimeSeriesFeatureCollection subset(ucar.unidata.geoloc.LatLonRect boundingBox) throws IOException {
    return subset( getStationFeatures(boundingBox));
  }

  @Override
  public StationTimeSeriesFeatureCollection subset(LatLonRect boundingBox, CalendarDateRange dateRange) throws IOException {
    return subset(getStationFeatures(boundingBox), dateRange);
  }

  @Override
  public StationTimeSeriesFeatureCollection subset(List<StationFeature> stations) throws IOException {
    return new StationSubset(this, stations);
  }

  @Override
  public StationTimeSeriesFeatureCollection subset(List<StationFeature> stnsWanted, CalendarDateRange dateRange) throws IOException {
    if (dateRange == null)
      return subset(stnsWanted);

    List<StationFeature> subsetStations = new ArrayList<>();
    for (StationFeature sf : stnsWanted) {
      StationTimeSeriesFeature stsf = (StationTimeSeriesFeature) sf; // LOOK
      StationTimeSeriesFeature subset = stsf.subset(dateRange);
      subsetStations.add(subset);
    }

    return new StationSubset(this, subsetStations);
  }

  @Override
  public StationTimeSeriesFeatureCollection subsetFeatures(List<StationFeature> stationsFeatures) throws IOException {
    return new StationSubset(this, stationsFeatures);
  }

  private static class StationSubset extends StationTimeSeriesCollectionImpl {
    private final List<StationFeature> stations;

    StationSubset(StationTimeSeriesCollectionImpl from, List<StationFeature> stations) {
      super(from.getName(), from.getTimeUnit(), from.getAltUnits());
      this.stations = stations;
    }

    @Override
    protected StationHelper createStationHelper() throws IOException {
      StationHelper stationHelper = new StationHelper();
      stationHelper.setStations(stations);
      return stationHelper;
    }

    @Override
    public List<StationFeature> getStationFeatures() throws IOException {
      return getStationHelper().getStationFeatures();
    }
  }

  ///////////////////////////////
  // flatten

  // might need to override for efficiency
  @Override
  public PointFeatureCollection flatten(List<String> stationNames, CalendarDateRange dateRange, List<VariableSimpleIF> varList) throws IOException {
    if ((stationNames == null) || (stationNames.size() == 0))
      return new StationTimeSeriesCollectionFlattened(this, dateRange);

    List<StationFeature> subsetStations = getStationHelper().getStationFeaturesFromNames(stationNames);
    return new StationTimeSeriesCollectionFlattened(new StationSubset(this, subsetStations), dateRange);
  }

  @Override
  public PointFeatureCollection flatten(LatLonRect boundingBox, CalendarDateRange dateRange) throws IOException {
    if (boundingBox == null)
      return new StationTimeSeriesCollectionFlattened(this, dateRange);

    List<StationFeature> subsetStations = getStationHelper().getStationFeatures(boundingBox);
    return new StationTimeSeriesCollectionFlattened(new StationSubset(this, subsetStations), dateRange);
  }

  @Override
  public StationFeature getStationFeature(PointFeature flatPointFeature) throws IOException {
    if (flatPointFeature instanceof StationFeatureHas) {
      return ((StationFeatureHas)flatPointFeature).getStationFeature();
    }
    return null;
  }

  /////////////////////////////////////////////////////////////////////////////////////

  @Override
  public Iterator<StationTimeSeriesFeature> iterator() {
    try {
      PointFeatureCollectionIterator pfIterator = getPointFeatureCollectionIterator();
      return new CollectionIteratorAdapter<>(pfIterator);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public IOIterator<PointFeatureCollection> getCollectionIterator() throws IOException {
    return new StationIterator();
  }

  @Override
  public PointFeatureCollectionIterator getPointFeatureCollectionIterator() throws IOException {
    return new StationIterator();
  }

  // note this assumes that a StationFeature is-a PointFeatureCollection (meaning is-a StationTimeSeriesFeature) (see the cast in next())
  // subclasses must override if thats not true
  // note that subset() may have made a subset of stationHelper
  private class StationIterator implements PointFeatureCollectionIterator, IOIterator<PointFeatureCollection> {
    Iterator<StationFeature> stationIter;
    DsgCollectionImpl prev;
    CollectionInfo calcInfo;

    public StationIterator() {
      stationIter = getStationHelper().getStationFeatures().iterator();
      CollectionInfo info = getInfo();
      if (!info.isComplete())
        calcInfo = info;
    }

    @Override
    public boolean hasNext() throws IOException {
      if (prev != null && calcInfo != null)
        calcInfo.extend(prev.getInfo());

      boolean more = stationIter.hasNext();
      if (!more && calcInfo != null) calcInfo.setComplete();
      return more;
    }

    @Override
    public PointFeatureCollection next() throws IOException {
      PointFeatureCollection result = (PointFeatureCollection) stationIter.next(); // LOOK
      prev = (DsgCollectionImpl) result; // common for Station and StationProfile
      return result;
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////
  // deprecated

  protected PointFeatureCollectionIterator localIterator;

  @Override
  public boolean hasNext() throws IOException {
    if (localIterator == null) resetIteration();
    return localIterator.hasNext();
  }

  @Override
  public void finish() {
    if (localIterator != null)
      localIterator.close();
  }

  @Override
  public StationTimeSeriesFeature next() throws IOException {
    return (StationTimeSeriesFeature) localIterator.next();
  }

  @Override
  public void resetIteration() throws IOException {
    localIterator = getPointFeatureCollectionIterator();
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.catalog;

import ucar.nc2.units.DateType;

import java.util.*;
import java.net.*;

/**
 * Public interface to a thredds catalog, a virtual directory of datasets.
 * A catalog consists of nested collections of InvDatasets.
 *
 * @author john caron
 */

public abstract class InvCatalog {
  protected String name, version;
  protected URI baseURI = null;
  protected InvDatasetImpl topDataset;
  protected Map<String, InvDataset> dsHash = new HashMap<String, InvDataset>(); // for datasets with ids
  protected Map<String, InvService> serviceHash = new HashMap<String, InvService>(); // for services (unique by name)

  protected List<InvService> services = new ArrayList<InvService>(); // InvService
  protected List<InvProperty> properties = new ArrayList<InvProperty>(); // InvProperty
  protected List<InvDataset> datasets = new ArrayList<InvDataset>(); // InvDataset
  protected DateType expires;

  /**
   * Protected constructor.
   */
  protected InvCatalog() {
  }

  /**
   * Get the name of the catalog
   *
   * @return name of catalog
   */
  public String getName() {
    return name;
  }

  /**
   * Get the version of the catalog
   *
   * @return catalog version
   */
  public String getVersion() {
    return version;
  }

  /**
   * Get top-level dataset.
   *
   * @return top-level InvDataset. May not be null.
   * @deprecated use getDatasets()
   */
  public InvDatasetImpl getDataset() {
    return topDataset;
  }

  /**
   * Find a contained dataset by its ID.
   *
   * @param id : dataset ID, case sensitive
   * @return InvDataset or null if not found.
   */
  public InvDataset findDatasetByID(String id) {
    return dsHash.get(id);
  }

  /**
   * Get top-level datasets.
   *
   * @return List of InvDataset. May be empty, may not be null.
   */
  public java.util.List<InvDataset> getDatasets() {
    return datasets;
  }

  /**
   * Get top-level services.
   *
   * @return List of InvService. May be empty, may not be null.
   */
  public java.util.List<InvService> getServices() {
    return services;
  }

  /**
   * Get catalog properties.
   *
   * @return List of InvProperty. May be empty, may not be null.
   */
  public java.util.List<InvProperty> getProperties() {
    return properties;
  }

  /**
   * Find named property.
   *
   * @param name match this name
   * @return string value of property or null if not exist.
   */
  public String findProperty(String name) {
    InvProperty result = null;
    for (InvProperty p : getProperties()) {
      if (p.getName().equals(name))
        result = p;
    }
    return (result == null) ? null : result.getValue();
  }

  /**
   * Get date catalog expires, or null if none given.
   *
   * @return expiration date, or null
   */
  public DateType getExpires() {
    return expires;
  }

  /**
   * Find the named service declared in the top level of this catalog.
   *
   * @param name match this name
   * @return service that matches the given name, or null if none found.
   */
  public InvService findService(String name) {
    if (name == null) return null;

    for (InvService s : services) {
      if (name.equals(s.getName())) return s;

      // look for nested servers
      if (s.getServiceType() == ServiceType.COMPOUND) {
        InvService result = s.findNestedService(name);
        if (result != null) return result;
      }
    }

    return null;
  }

  /**
   * Resolve reletive URIs, using the catalog's base URI. If the uriString is not reletive, then
   * no resolution is done. This also allows baseURI to be a file: scheme.
   *
   * @param uriString any url, reletive or absolute
   * @return resolved url string, or null on error
   * @throws URISyntaxException if uriString violates RFC 2396
   * @see java.net.URI#resolve
   */
  public URI resolveUri(String uriString) throws URISyntaxException {
    URI want = new URI(uriString);

    if ((baseURI == null) || want.isAbsolute())
      return want;

    // gotta deal with file ourself
    String scheme = baseURI.getScheme();
    if ((scheme != null) && scheme.equals("file")) { // LOOK at ucar.nc2.util.NetworkUtils.resolve
      String baseString = baseURI.toString();
      if ((uriString.length() > 0) && (uriString.charAt(0) == '#'))
        return new URI(baseString + uriString);
      int pos = baseString.lastIndexOf('/');
      if (pos > 0) {
        String r = baseString.substring(0, pos + 1) + uriString;
        return new URI(r);
      }
    }

    //otherwise let the URI class resolve it
    return baseURI.resolve(want);
  }

  /**
   * Get the base URI as a String
   *
   * @return baseURI as a String
   */
  public String getUriString() {
    return baseURI.toString();
  }

  /**
   * Check internal data structures.
   *
   * @param out  : print errors here
   * @param show : print messages for each object (debug)
   * @return true if no fatal consistency errors.
   */
  abstract public boolean check(StringBuilder out, boolean show);

  /**
   * Check internal data structures.
   *
   * @param out : print errors here
   * @return true if no fatal consistency errors.
   */
  public boolean check(StringBuilder out) {
    return check(out, false);
  }

  /**
   * Munge this catalog so the given dataset is the top catalog.
   *
   * @param dataset make this top; must be existing dataset in this catalog.
   */
  abstract public void subset(InvDataset dataset);

  /**
   * Munge this catalog to remove any dataset that doesnt pass through the filter.
   *
   * @param filter remove datasets that dont pass this filter.
   */
  abstract public void filter(DatasetFilter filter);

}