/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.httpservices;

import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.auth.AuthScope;
import org.apache.http.client.methods.HttpOptions;
import org.apache.http.message.BasicHttpResponse;

import java.lang.reflect.Constructor;
import java.util.Set;

/**
 * HTTPFactory creates method instance.
 * This code was originally in HttpMethod.
 */

public class HTTPFactory
{
    // In order to test client side code that mocks
    // HTTPMethod, provide a static global
    // than can be set by a test program.

    static public java.lang.Class MOCKMETHODCLASS = null;

    //////////////////////////////////////////////////////////////////////////
    // Static factory methods for creating HTTPSession instances

    static public HTTPSession newSession(String host, int port) throws HTTPException
    {
        return new HTTPSession(host, port);
    }

    static public HTTPSession newSession(String url) throws HTTPException
    {
        return new HTTPSession(url);
    }

    static public HTTPSession newSession(HttpHost target) throws HTTPException
    {
        return new HTTPSession(target);
    }

    @Deprecated
    static public HTTPSession newSession(AuthScope scope) throws HTTPException
    {
        HttpHost hh = new HttpHost(scope.getHost(), scope.getPort(), null);
        return new HTTPSession(hh);
    }

    //////////////////////////////////////////////////////////////////////////
    // Static factory methods for creating HTTPMethod instances

    static public HTTPMethod Get(HTTPSession session, String legalurl) throws HTTPException
    {
        return makemethod(HTTPSession.Methods.Get, session, legalurl);
    }

    static public HTTPMethod Head(HTTPSession session, String legalurl) throws HTTPException
    {
        return makemethod(HTTPSession.Methods.Head, session, legalurl);
    }

    static public HTTPMethod Put(HTTPSession session, String legalurl) throws HTTPException
    {
        return makemethod(HTTPSession.Methods.Put, session, legalurl);
    }

    static public HTTPMethod Post(HTTPSession session, String legalurl) throws HTTPException
    {
        return makemethod(HTTPSession.Methods.Post, session, legalurl);
    }

    static public HTTPMethod Options(HTTPSession session, String legalurl) throws HTTPException
    {
        return makemethod(HTTPSession.Methods.Options, session, legalurl);
    }

    static public HTTPMethod Get(HTTPSession session) throws HTTPException
    {
        return Get(session, null);
    }

    static public HTTPMethod Head(HTTPSession session) throws HTTPException
    {
        return Head(session, null);
    }

    static public HTTPMethod Put(HTTPSession session) throws HTTPException
    {
        return Put(session, null);
    }

    static public HTTPMethod Post(HTTPSession session) throws HTTPException
    {
        return Post(session, null);
    }

    static public HTTPMethod Options(HTTPSession session) throws HTTPException
    {
        return Options(session, null);
    }

    static public HTTPMethod Get(String legalurl) throws HTTPException
    {
        return Get(null, legalurl);
    }

    static public HTTPMethod Head(String legalurl) throws HTTPException
    {
        return Head(null, legalurl);
    }

    static public HTTPMethod Put(String legalurl) throws HTTPException
    {
        return Put(null, legalurl);
    }

    static public HTTPMethod Post(String legalurl) throws HTTPException
    {
        return Post(null, legalurl);
    }

    static public HTTPMethod Options(String legalurl) throws HTTPException
    {
        return Options(null, legalurl);
    }

    /**
     * Common method creation code so we can isolate mocking
     *
     * @param session
     * @return
     * @throws HTTPException
     */
    static protected HTTPMethod makemethod(HTTPSession.Methods m, HTTPSession session, String url)
            throws HTTPException
    {
        HTTPMethod meth = null;
        if(MOCKMETHODCLASS == null) { // do the normal case
            meth = new HTTPMethod(m, session, url);
        } else {//(MOCKMETHODCLASS != null)
            java.lang.Class methodcl = MOCKMETHODCLASS;
            Constructor<HTTPMethod> cons = null;
            try {
                cons = methodcl.getConstructor(HTTPSession.Methods.class, HTTPSession.class, String.class);
            } catch (Exception e) {
                throw new HTTPException("HTTPFactory: no proper HTTPMethod constructor available", e);
            }
            try {
                meth = cons.newInstance(m, session, url);
            } catch (Exception e) {
                throw new HTTPException("HTTPFactory: HTTPMethod constructor failed", e);
            }
        }
        return meth;
    }

    static public Set<String> getAllowedMethods()
    {
        HttpResponse rs = new BasicHttpResponse(new ProtocolVersion("http", 1, 1), 0, "");
        Set<String> set = new HttpOptions().getAllowedMethods(rs);
        return set;
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.hdf5;

import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;

import java.io.File;
import java.io.IOException;
import java.util.Formatter;

/**
 * HDF5 diagnostic helper
 *
 * @author caron
 * @since 6/25/12
 */
public class H5diag {
  private H5iosp iosp;

  public H5diag(H5iosp iosp) {
    this.iosp = iosp;
  }

  private static class Size {
    long storage;
    long nominal;
    int count;

    private Size(long storage, int count) {
      this.storage = storage;
      this.count = count;
    }

    float getRatio() {
      if (storage == 0) return 0;
      return ((float) nominal / storage) ;
    }
  }

  public void showCompress(Formatter f) throws IOException {
    NetcdfFile ncfile = iosp.getNetcdfFile();

    Size totalSize = new Size(0,0);
    for (Variable v : ncfile.getVariables())  {
      H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
      showCompress(v, vinfo, totalSize, f);
    }
    f.format("%n");
    f.format(" total bytes   = %d%n", totalSize.nominal);
    f.format(" total storage = %d%n", totalSize.storage);
    f.format(" compression = %f%n", totalSize.getRatio());
    f.format(" nchunks     = %d%n", totalSize.count);

    File raf = new File(ncfile.getLocation());
    f.format(" file size    = %d%n", raf.length());
    float overhead = totalSize.storage == 0 ? 0 : ((float) raf.length()/totalSize.storage);
    f.format(" overhead     = %f%n", overhead);
  }

  public void showCompress(Variable v, H5header.Vinfo vinfo, Size total, Formatter f) throws IOException {
    H5header.MessageDataspace mdt = vinfo.mds;

    long total_elems = 1;
    f.format("%8s %-40s(", v.getDataType(), v.getShortName());
    for (int len : mdt.dimLength) {
      f.format("%d ", len);
      total_elems *= len;
    }
    boolean sizeOk = total_elems == v.getSize();
    total_elems = v.getSize();

    long nominalSize = total_elems * v.getElementSize();

    Size size = new Size(nominalSize, 1);
    countStorageSize(vinfo, size);
    total.storage += size.storage;
    total.nominal += nominalSize;
    total.count += size.count;
    float ratio =  (size.storage == 0) ? 0 : (float) nominalSize / size.storage;

    f.format(") == %d nelems %s == %d bytes storage = %d (%f) nchunks = %d%n", total_elems, sizeOk ? "" : "*", nominalSize, size.storage, ratio, size.count);
  }

  private void countStorageSize(H5header.Vinfo vinfo, Size size) throws IOException {
    DataBTree btree = vinfo.btree;
    if (btree == null || vinfo.useFillValue) {
      size.storage = 0;
      size.count = 0;
      return; // 0 storage
    }

    int count = 0;
    long total = 0;
    DataBTree.DataChunkIterator iter = btree.getDataChunkIteratorFilter(null);
    while (iter.hasNext()) {
      DataBTree.DataChunk dc = iter.next();
      total += dc.size;
      count++;
    }

    size.storage = total;
    size.count = count;
  }

  public long[] countStorageSize(H5header.Vinfo vinfo, Size size, Formatter f) throws IOException {
    long[] result = new long[2];
    DataBTree btree = vinfo.btree;
    if (btree == null) {
      if (f != null) f.format("btree is null%n");
      return result;
    }
    if (vinfo.useFillValue) {
      if (f != null) f.format("useFillValue - no data is stored%n");
      return result;
    }

    int count = 0;
    long total = 0;
    DataBTree.DataChunkIterator iter = btree.getDataChunkIteratorFilter(null);
    while (iter.hasNext()) {
      DataBTree.DataChunk dc = iter.next();
      if (f != null) f.format(" %s%n", dc);
      total += dc.size;
      count++;
    }

    result[0] = total;
    result[1] = count;
    return result;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////

  public void deflate(Formatter f, Variable v) throws IOException {
    H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
    DataBTree btree = vinfo.btree;
    if (btree == null || vinfo.useFillValue) {
      f.format("%s not chunked%n", v.getShortName());
      return;
    }
  }

  public void deflate(Formatter f, DataBTree btree) throws IOException {

    int count = 0;
    long total = 0;
    DataBTree.DataChunkIterator iter = btree.getDataChunkIteratorFilter(null);
    while (iter.hasNext()) {
      DataBTree.DataChunk dc = iter.next();
      total += dc.size;
      count++;
    }

  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import javax.annotation.Nullable;
import thredds.featurecollection.FeatureCollectionConfig;
import ucar.nc2.time.CalendarDate;
import ucar.unidata.io.RandomAccessFile;

import java.util.List;

/**
 * Read Grib1Partition from ncx index
 *
 * @author caron
 * @since 2/21/14
 */
class Grib1PartitionBuilderFromIndex extends Grib1CollectionBuilderFromIndex {

  // read in the index, index raf already open; return null on failure
  @Nullable
  static Grib1Partition createTimePartitionFromIndex(String name, RandomAccessFile raf,
      FeatureCollectionConfig config, org.slf4j.Logger logger) {

    Grib1PartitionBuilderFromIndex builder = new Grib1PartitionBuilderFromIndex(name, config, logger);
    if (builder.readIndex(raf))
      return new Grib1Partition(builder.pc);

    return null;
  }

  // read in the index, index raf already open; return null on failure
  @Nullable
  static PartitionCollectionMutable openMutablePCFromIndex(String name, RandomAccessFile raf,
      FeatureCollectionConfig config, org.slf4j.Logger logger) {

    Grib1PartitionBuilderFromIndex builder = new Grib1PartitionBuilderFromIndex(name, config, logger);
    if (builder.readIndex(raf))
      return builder.pc;

    return null;
  }

  //////////////////////////////////////////////////////////////////////////////////

  private final PartitionCollectionMutable pc;  // build this object

  private Grib1PartitionBuilderFromIndex(String name, FeatureCollectionConfig config, org.slf4j.Logger logger) {
    super(name, config, logger);
    this.pc = new PartitionCollectionMutable(name, null, config, true, logger);
    this.gc = pc;
  }

  @Override
  public String getMagicStart() {
    return Grib1PartitionBuilder.MAGIC_START;
  }

  ///////////////////////////////////////////////////////////////////////////
  // reading ncx

  /*
  extend GribCollection {
    repeated Partition partitions = 100;
    required bool isPartitionOfPartitions = 101;
    repeated uint32 run2part = 102;       // masterRuntime index to partition index
  }
   */
  @Override
  protected boolean readExtensions(GribCollectionProto.GribCollection proto) {
    pc.isPartitionOfPartitions = proto.getIsPartitionOfPartitions();

    List<Integer> list = proto.getRun2PartList();
    pc.run2part = new int[list.size()];
    int count = 0;
    for (int partno : list)
      pc.run2part[count++] = partno;

    List<ucar.nc2.grib.collection.GribCollectionProto.Partition> partList = proto.getPartitionsList();
    for (ucar.nc2.grib.collection.GribCollectionProto.Partition partProto : proto.getPartitionsList())
      makePartition(partProto);

    return partList.size() > 0;
  }

  /*
  extend Variable {
    repeated PartitionVariable partition = 100;
    repeated Parameter vparams = 101;    // not used yet
  }
   */
  @Override
  protected GribCollectionMutable.VariableIndex readVariableExtensions(GribCollectionMutable.GroupGC group, GribCollectionProto.Variable proto, GribCollectionMutable.VariableIndex vi) {
    List<GribCollectionProto.PartitionVariable> pvList = proto.getPartVariableList();
    PartitionCollectionMutable.VariableIndexPartitioned vip = pc.makeVariableIndexPartitioned(group, vi, pvList.size());
    vip.setPartitions(pvList);

    // cant put this in the constructor
    vip.ndups = vi.ndups;
    vip.nrecords = vi.nrecords;
    vip.nmissing = vi.nmissing;
    return vip;
  }

  /*
message Partition {
  required string name = 1;       // name is used in TDS - eg the subdirectory when generated by TimePartitionCollections
  required string filename = 2;   // the gribCollection.ncx2 file
  required string directory = 3;   // top directory
  optional uint64 lastModified = 4;
}
   */
  private PartitionCollectionMutable.Partition makePartition(GribCollectionProto.Partition proto) {
    long partitionDateMillisecs =  proto.getPartitionDate();
    CalendarDate partitionDate =  partitionDateMillisecs > 0 ? CalendarDate.of(partitionDateMillisecs) : null;
    return pc.addPartition(proto.getName(), proto.getFilename(), proto.getLastModified(), proto.getLength(), null, partitionDate);
  }
}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.servlet;

import dap4.core.dmr.DapEnumConst;
import dap4.core.dmr.DapEnumeration;
import dap4.core.dmr.DapType;
import dap4.core.dmr.TypeSort;
import dap4.core.util.DapException;

import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Random;

public class RandomValue extends Value
{
    //////////////////////////////////////////////////
    // Constants

    static final long SEED = 37L;

    //////////////////////////////////////////////////
    // Instance variables

    protected Random random = new Random(SEED);

    //////////////////////////////////////////////////
    // Constructor(s)

    public RandomValue()
    {
    }

    //////////////////////////////////////////////////
    // Accessors

    //////////////////////////////////////////////////
    // IValue Interface

    public ValueSource source()
    {
        return ValueSource.RANDOM;
    }

    public Object
    nextValue(DapType basetype)
            throws DapException
    {
        Object value = null;
        TypeSort atomtype = basetype.getTypeSort();
        if(atomtype.isIntegerType())
            value = nextInteger(basetype);
        else if(atomtype.isFloatType())
            value = nextFloat(basetype);
        else switch (atomtype) {
            case Char:
                value = new char[]{nextString(random, 1, 32, 127).charAt(0)};
                break;
            case String:
                value = new String[]{nextString(random, MAXSTRINGSIZE, 32, 127)};
                break;
            case URL:
                value = new String[]{nextURL()};
                break;
            case Opaque:
                int length = 2 + (random.nextInt(MAXOPAQUESIZE) * 2);
                byte[] bytes = new byte[length];
                random.nextBytes(bytes);
                value = new ByteBuffer[]{ByteBuffer.wrap(bytes)};// order is irrelevant
                break;
            case Enum:
                //Coverity[FB.BC_UNCONFIRMED_CAST]
                value = nextEnum(((DapEnumeration) basetype));
                break;
            case Int8:
            case UInt8:
            case Int16:
            case UInt16:
            case Int32:
            case UInt32:
            case Int64:
            case UInt64:
            case Float32:
            case Float64:
            default:
                throw new DapException("Unexpected type: " + basetype);
            }
        if(DEBUG) {
            System.err.printf("RandomValue.nextValue: (%s) %s",
                    atomtype.toString(), value.toString());
            if(atomtype.isIntegerType()) {
                Number nn = (Number) value;
                System.err.printf(" 0x%x\n", nn.longValue());
            } else
                System.err.println();
        }
        return value;
    }

    // return an integer type value (including long, but not floats)
    public Object
    nextInteger(DapType basetype)
            throws DapException
    {
        TypeSort atomtype = basetype.getTypeSort();
        if(!atomtype.isIntegerType())
            throw new DapException("Unexpected type: " + basetype);
        boolean unsigned = atomtype.isUnsigned();
        switch (atomtype) {
        case Int8:
            return new byte[]{(byte) (random.nextInt(1 << 8) - (1 << 7))};
        case UInt8:
            return new byte[]{(byte) (random.nextInt(1 << 8) & 0xFF)};
        case Int16:
            return new short[]{(short) (random.nextInt(1 << 16) - (1 << 15))};
        case UInt16:
            return new short[]{(short) (random.nextInt(1 << 16))};
        case Int32:
            return new int[]{random.nextInt()};
        case UInt32:
            long l = random.nextLong();
            l = l & 0xFFFFFFFF;
            return new int[]{(int) l};
        case Int64:
            return new long[]{random.nextLong()};
        case UInt64:
            return new long[]{new BigInteger(64, random).longValue()};
        }
        throw new DapException("Unexpected type: " + basetype);
    }

    // return a float type value
    public Object
    nextFloat(DapType basetype)
            throws DapException
    {
        TypeSort atomtype = basetype.getTypeSort();
        switch (atomtype) {
        case Float32:
            return new float[]{random.nextFloat()};
        case Float64:
            return new double[]{random.nextDouble()};
        default:
            break;
        }
        throw new DapException("Unexpected type: " + basetype);
    }

    Object
    nextEnum(DapEnumeration en)
    {
        long l;
        TypeSort basetype = en.getBaseType().getTypeSort();

        // Collect the enum const values as BigIntegers
        List<String> ecnames = en.getNames();
        BigInteger[] econsts = new BigInteger[ecnames.size()];
        for(int i = 0; i < econsts.length; i++) {
            DapEnumConst ec = en.lookup(ecnames.get(i));
            econsts[i] = BigInteger.valueOf(ec.getValue());
            if(basetype == TypeSort.UInt64)
                econsts[i] = econsts[i].and(MASK);
        }

        int index = random.nextInt(econsts.length);
        l = econsts[index].longValue();
        Object val = null;
        switch (basetype) {
        case Int8:
            val = new byte[]{(byte) l};
            break;
        case UInt8:
            val = new byte[]{(byte) (l & 0xFFL)};
            break;
        case Int16:
            val = new short[]{(short) l};
            break;
        case UInt16:
            val = new short[]{(short) (l & 0xFFFFL)};
            break;
        case Int32:
            val = new int[]{(int) l};
            break;
        case UInt32:
            val = new int[]{(int) (l & 0xFFFFFFFFL)};
            break;
        case Int64:
            val = new long[]{l};
            break;
        case UInt64:
            val = new long[]{l};
            break;
        }
        return val;
    }

    String
    nextString(Random random, int maxlength, int min, int max)
    {
        int length = random.nextInt(maxlength) + 1;
        StringBuilder buf = new StringBuilder();
        if(asciionly && max > 127) max = 127;
        int range = (max + 1) - min; // min..max+1 -> 0..(max+1)-min
        for(int i = 0; i < length; i++) {
            int c = random.nextInt(range); // 0..(max+1)-min
            c = c + min; // 0..(max+1)-min -> min..max+1
            buf.append((char) c);
        }
        return buf.toString();
    }

    static final String[] protocols = new String[]{"http", "https"};
    static final String legal =
            "abcdefghijklmnoqqrstuvwxyz"
                    + "ABCDEFGHIJKLMNOQQRSTUVWXYZ"
                    + "0123456789"
                    + "_";

    String
    nextURL()
    {
        StringBuilder url = new StringBuilder();
        url.append(protocols[random.nextInt(protocols.length)]);
        url.append("://");
        for(int i = 0; i < HOSTNSEG; i++) {
            if(i > 0) url.append(".");
            for(int j = 0; j < MAXSEGSIZE; j++) {
                int c;
                do {
                    c = random.nextInt('z');
                } while(legal.indexOf(c) < 0);
                url.append((char) c);
            }
        }
        if(random.nextBoolean())
            url.append(String.format(":%d", random.nextInt(5000) + 1));
        for(int i = 0; i < PATHNSEG; i++) {
            url.append("/");
            for(int j = 0; j < MAXSEGSIZE; j++) {
                int c;
                do {
                    c = random.nextInt('z');
                } while(legal.indexOf(c) < 0);
                url.append((char) c);
            }
        }
        return url.toString();
    }

    /**
     * Return an integer in range 1..max inclusive.
     *
     * @param max
     * @return random integer in range
     * @throws DapException
     */
    public int
    nextCount(int max)
            throws DapException
    {
        int min = 1;
        if(max < min || min < 1)
            throw new DapException("bad range");
        int range = (max + 1) - min;  // min..max+1 -> 0..(max+1)-min
        int n = random.nextInt(range);   //  0..(max+1)-min
        n = n + min;   // min..(max+1)
        if(DEBUG)
            System.err.println("RandomValue.nextCount: " + n);
        return n;
    }

}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.cdm.nc2;

import dap4.cdm.CDMUtil;
import dap4.cdm.NodeMap;
import dap4.core.data.DSP;
import dap4.core.data.DSPRegistry;
import dap4.core.util.DapContext;
import dap4.core.util.DapException;
import dap4.core.util.DapUtil;
import dap4.dap4lib.DapCodes;
import dap4.dap4lib.FileDSP;
import dap4.dap4lib.HttpDSP;
import dap4.dap4lib.XURI;
import ucar.ma2.*;
import ucar.nc2.NetcdfFile;
import ucar.nc2.ParsedSectionSpec;
import ucar.nc2.Variable;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.iosp.IospHelper;
import ucar.nc2.util.CancelTask;

import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.channels.WritableByteChannel;
import java.util.*;

public class DapNetcdfFile extends NetcdfFile
{
    static final boolean DEBUG = false;
    static final boolean PARSEDEBUG = false;
    static final boolean MERGE = false;

    // NetcdfDataset enhancement to use: need only coord systems
    static Set<NetcdfDataset.Enhance> ENHANCEMENT = EnumSet.of(NetcdfDataset.Enhance.CoordSystems);

    //////////////////////////////////////////////////
    // Constants
    static final String QUERYSTART = "?";
    static final String FRAGSTART = "#";

    //////////////////////////////////////////////////
    // Type Declarations

    static protected class NullCancelTask implements CancelTask
    {
        public boolean isCancel()
        {
            return false;
        }

        public void setError(String msg)
        {
        }

        public void setProgress(String msg, int progress)
        {
        }

    }

    //////////////////////////////////////////////////
    // Static variables

    static protected final NullCancelTask nullcancel = new NullCancelTask();

    /**
     * Define a map of known DSP classes.
     */
    static protected DSPRegistry dspregistry = new DSPRegistry();

    static {
        dspregistry.register(FileDSP.class, DSPRegistry.FIRST);
        dspregistry.register(HttpDSP.class, DSPRegistry.FIRST);
    }

    //////////////////////////////////////////////////
    // Instance Variables

    protected boolean allowCompression = true;
    protected boolean closed = false;

    protected String location = null; // original argument passed to open
    protected String dsplocation = null; // what is passed to DSP
    protected XURI xuri = null;
    protected DSP dsp = null;

    protected CancelTask cancel = null;

    // protected NodeMap nodemap = null;  unused?

    /**
     * Originally, the array for a variable was stored
     * using var.setCacheData(). However, that is illegal
     * for Structures and Sequences, so (for now)
     * we maintain a map variable->array.
     */
    protected Map<Variable, Array> arraymap = new HashMap<>();

    //////////////////////////////////////////////////
    // Constructor(s)

    /**
     * Open a Dap4 connection or file via a D4DSP.
     *
     * @param location   URL for the request. Note that if this is
     *                   intended to send to a file oriented
     *                   DSP, then if must be converted to an absolute path.
     * @param cancelTask check if task is cancelled; may be null.
     * @throws IOException
     */
    public DapNetcdfFile(String location, CancelTask cancelTask)
            throws IOException
    {
        super();
        this.location = location;
        // Figure out the location: url vs path
        XURI xuri;
        try {
            xuri = new XURI(location);
        } catch (URISyntaxException use) {
            throw new IOException(use);
        }
        boolean isfile = xuri.isFile();
        if(isfile) {
            this.dsplocation = DapUtil.absolutize(xuri.getPath());
        } else { // Not a file url
            this.dsplocation = xuri.assemble(XURI.URLBASE);
        }
        DapContext cxt = new DapContext();
        cancel = (cancelTask == null ? nullcancel : cancelTask);
        // 1. Get and parse the constrained DMR and Data v-a-v URL
        this.dsp = dspregistry.findMatchingDSP(location,cxt); // will set dsp context
        if(this.dsp == null)
            throw new IOException("No matching DSP: "+this.location);
        this.dsp.setContext(cxt);
        this.dsp.open(this.dsplocation);

        // 2. Construct an equivalent CDM tree and populate
        //    this NetcdfFile object.
        CDMCompiler compiler = new CDMCompiler(this, this.dsp);
        compiler.compile();
        // set the pseudo-location, otherwise we get a name that is full path.
        setLocation(this.dsp.getDMR().getDataset().getShortName());
        finish();
        this.arraymap = compiler.getArrayMap();
    }

    /**
     * Open a Dap4 connection
     *
     * @param url URL for the request.
     * @throws IOException
     */

    public DapNetcdfFile(String url)
            throws IOException
    {
        this(url, null);
    }

    //////////////////////////////////////////////////
    // Close

    /**
     * Close all resources (files, sockets, etc) associated with this file.
     *
     * @throws java.io.IOException if error when closing
     */
    @Override
    public synchronized void close()
            throws java.io.IOException
    {
        if(closed) return;
        closed = true; // avoid circular calls
        dsp = null;
        // nodemap = null;  unused?
    }

    //////////////////////////////////////////////////
    // Accessors

    /**
     * @return true if we can ask the server to do constraint processing
     */
    public boolean isconstrainable()
    {
        return true;
    }

    public String getLocation()
    {
        return location;
    }

    public DSP getDSP()
    {
        return this.dsp;
    }

    //////////////////////////////////////////////////
    // Override NetcdfFile.readXXX Methods

    /**
     * Do a bulk read on a list of Variables and
     * return a corresponding list of Array that contains the results
     * of a full read on each Variable.
     * TODO: optimize to make only a single server call and cache the results.
     *
     * @param variables List of type Variable
     * @return List of Array, one for each Variable in the input.
     * @throws IOException if read error
     */

    @Override
    public List<Array>
    readArrays(List<Variable> variables)
            throws IOException
    {
        List<Array> result = new ArrayList<Array>();
        for(Variable variable : variables) {
            result.add(variable.read());
        }
        return result;
    }

    /**
     * Read databuffer from a top level Variable
     * and send databuffer to a WritableByteChannel.
     * Experimental.
     *
     * @param v       a top-level Variable
     * @param section the section of databuffer to read.
     *                There must be a Range for each Dimension in the variable,
     *                in order.
     *                Note: no nulls allowed. IOSP may not modify.
     * @param channel write databuffer to this WritableByteChannel
     * @return the number of databuffer written to the channel
     * @throws java.io.IOException            if read error
     * @throws ucar.ma2.InvalidRangeException if invalid section
     */

    @Override
    public long readToByteChannel(Variable v, Section section, WritableByteChannel channel)
            throws java.io.IOException, ucar.ma2.InvalidRangeException
    {
        Array result = readData(v, section);
        return IospHelper.transferData(result, channel);
    }

    public Array
    readSection(String variableSection)
            throws IOException, InvalidRangeException
    {
        ParsedSectionSpec cer
                = ParsedSectionSpec.parseVariableSection(this, variableSection);
        return cer.v.read(cer.section);
    }

    /**
     * Primary read entry point.
     * This is the primary implementor of Variable.read.
     *
     * @param cdmvar  A top-level variable
     * @param section the section of databuffer to read.
     *                There must be a Range for each Dimension in the variable,
     *                in order. Note: no nulls allowed.
     * @return An Array object for accessing the databuffer
     * @throws IOException           if read error
     * @throws InvalidRangeException if invalid section
     */

    @Override
    protected Array
    readData(Variable cdmvar, Section section)
            throws IOException, InvalidRangeException
    {
        // The section is applied wrt to the DataDMR, so it
        // takes into account any constraint used in forming the dataDMR.
        // We use the Section to produce a view of the underlying variable array.

        assert this.dsp != null;
        Array result = arraymap.get(cdmvar);
        if(result == null)
            throw new IOException("No data for variable: " + cdmvar.getFullName());
        if(section != null) {
            if(cdmvar.getRank() != section.getRank())
                throw new InvalidRangeException(String.format("Section rank != %s rank", cdmvar.getFullName()));
            List<Range> ranges = section.getRanges();
            // Case out the possibilities
            if(CDMUtil.hasVLEN(ranges)) {
                ranges = ranges.subList(0, ranges.size() - 1);// may produce empty list
            }
            if(ranges.size() > 0 && !CDMUtil.isWhole(ranges, cdmvar))
                result = result.sectionNoReduce(ranges);
        }
        return result;
    }
}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import java.io.*;

/**
 * Holds a OPeNDAP Int32 value.
 *
 * @author jehamby
 * @version $Revision: 15901 $
 * @see BaseType
 */
public class DInt32 extends DPrimitive implements ClientIO {
    /**
     * Constructs a new DInt32.
     */
    public DInt32() {
        super();
    }

    /**
     * Constructs a new DInt32 with name n.
     *
     * @param n the name of the variable.
     */
    public DInt32(String n) {
        super(n);
    }

    /**
     * The value of this DInt32.
     */
    private int val;

    /**
     * Get the current value as an int.
     *
     * @return the current value.
     */
    public final int getValue() {
        return val;
    }

    /**
     * Set the current value.
     *
     * @param newVal the new value.
     */
    public final void setValue(int newVal) {
        val = newVal;
    }

    /**
     * Constructs a new Int32PrimitiveVector.
     *
     * @return a new Int32PrimitiveVector.
     */
    public PrimitiveVector newPrimitiveVector() {
        return new Int32PrimitiveVector(this);
    }

    /**
     * Returns the OPeNDAP type name of the class instance as a String.
     *
     * @return the OPeNDAP type name of the class instance as a String.
     */
    public String getTypeName() {
        return "Int32";
    }

    /**
     * Prints the value of the variable, with its declaration.  This
     * function is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *
     * @param os           the PrintWriter on which to print the value.
     * @param space        this value is passed to the printDecl method,
     *                     and controls the leading spaces of the output.
     * @param print_decl_p a boolean value controlling whether the
     *                     variable declaration is printed as well as the value.
     * @see BaseType#printVal(PrintWriter, String, boolean)
     */
    public void printVal(PrintWriter os, String space, boolean print_decl_p) {
        if (print_decl_p) {
            printDecl(os, space, false);
            os.println(" = " + val + ";");
        } else
            os.print(val);
    }

    /**
     * Reads data from a DataInputStream. This method is only used
     * on the client side of the OPeNDAP client/server connection.
     *
     * @param source   a DataInputStream to read from.
     * @param sv       the ServerVersion returned by the server.
     * @param statusUI the StatusUI object to use for GUI updates
     *                 and user cancellation notification (may be null).
     * @throws EOFException if EOF is found before the variable is completely
     *                      deserialized.
     * @throws IOException  thrown on any other InputStream exception.
     * @see ClientIO#deserialize(DataInputStream, ServerVersion, StatusUI)
     */
    public synchronized void deserialize(DataInputStream source,
                                         ServerVersion sv,
                                         StatusUI statusUI)
            throws IOException, EOFException {
        val = source.readInt();
        if (statusUI != null)
            statusUI.incrementByteCount(4);
    }

    /**
     * Writes data to a DataOutputStream. This method is used
     * primarily by GUI clients which need to download OPeNDAP data, manipulate
     * it, and then re-save it as a binary file.
     *
     * @param sink a DataOutputStream to write to.
     * @throws IOException thrown on any OutputStream
     *                     exception.
     */
    public void externalize(DataOutputStream sink) throws IOException {
        sink.writeInt(val);
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.server.radarServer2;

import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;
import ucar.nc2.constants.CDM;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.*;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Handle configuration for the Radar Server
 */
public class RadarServerConfig {
    static ConcurrentHashMap<String, FileSystemProvider> fsproviders = new ConcurrentHashMap<>();

    static public List<RadarConfigEntry> readXML(String filename) {
        List<RadarConfigEntry> configs = new ArrayList<>();

        SAXBuilder builder = new SAXBuilder();
        File f = new File(filename);
        try {
            Document doc = builder.build(f);
            Element cat = doc.getRootElement();
            Namespace catNS = cat.getNamespace();
            Element topDS = cat.getChild("dataset", cat.getNamespace());
            for (Element dataset: topDS.getChildren("datasetScan", catNS)) {
                RadarConfigEntry conf = new RadarConfigEntry();
                configs.add(conf);

                Element collection = dataset.getChild("radarCollection", catNS);
                conf.dateParseRegex = collection.getAttributeValue("dateRegex");
                conf.dateFmt = collection.getAttributeValue("dateFormat");
                conf.layout = collection.getAttributeValue("layout");
                String crawlItems = collection.getAttributeValue("crawlItems");
                if (crawlItems != null) {
                    conf.crawlItems = Integer.parseInt(crawlItems);
                } else {
                    conf.crawlItems = 5;
                }

                Element meta = dataset.getChild("metadata", catNS);
                conf.name = dataset.getAttributeValue("name");
                conf.urlPath = dataset.getAttributeValue("path");
                conf.dataPath = getPath(dataset.getAttributeValue("location"));
                conf.dataFormat = meta.getChild("dataFormat", catNS).getValue();
                conf.stationFile = meta.getChild("stationFile", catNS).getAttributeValue("path");
                conf.doc = meta.getChild("documentation", catNS).getValue();

                conf.timeCoverage = readTimeCoverage(
                        meta.getChild("timeCoverage", catNS));

                conf.spatialCoverage = readGeospatialCoverage(
                        meta.getChild("geospatialCoverage", catNS));

                Element variables = meta.getChild("variables", catNS);
                conf.vars = new ArrayList<>();
                for (Element var: variables.getChildren("variable", catNS)) {
                    RadarConfigEntry.VarInfo inf = new RadarConfigEntry.VarInfo();
                    conf.vars.add(inf);

                    inf.name = var.getAttributeValue("name");
                    inf.vocabName = var.getAttributeValue("vocabulary_name");
                    inf.units = var.getAttributeValue("units");
                }
            }
        } catch (IOException | JDOMException e) {
            e.printStackTrace();
        }

        return configs;
    }

    private static Path getPath(String location) throws IOException {
        FileSystem fs;

        // If we're given an actual URI, use that to find the file system.
        // Otherwise, use the default.
        if(location.contains(":")) {
            URI uri = URI.create(location);

            // Fix parsing of s3:// (note two '/') style paths
            if (uri.getPath().isEmpty()) {
                uri = URI.create(location.replace("//", "///"));
            }

            location = uri.getPath();
            fs = getFS(uri);
        } else {
            fs = FileSystems.getDefault();
        }
        return fs.getPath(location);
    }

    private static FileSystem getFS(URI uri) throws IOException {
        FileSystem fs;

        try {
            fs = getProvider(uri).getFileSystem(uri);
        } catch (ProviderNotFoundException e) {
            System.out.println("Provider not found: " + e.getMessage());
            System.out.println("Using default file system.");
            fs = FileSystems.getDefault();
        }
        return fs;
    }

    // This is to work around the fact that when we *get* a filesystem, we
    // cannot pass in the class loader. This results in custom providers (say
    // S3) not being found. However, the filesystem already exists, so the
    // filesystem can't be re-created either.
    private static FileSystemProvider getProvider(URI uri) throws IOException {
        if(fsproviders.containsKey(uri.getScheme())) {
            return fsproviders.get(uri.getScheme());
        } else {
            FileSystem fs;
            try {
                fs = FileSystems.newFileSystem(uri,
                        new HashMap<String, Object>(),
                        Thread.currentThread().getContextClassLoader());
            } catch (FileSystemAlreadyExistsException e) {
                fs = FileSystems.getFileSystem(uri);
            }
            fsproviders.put(uri.getScheme(), fs.provider());
            return fs.provider();
        }
    }

    protected static RadarConfigEntry.GeoInfo readGeospatialCoverage(Element gcElem) {
        if (gcElem == null) return null;
        Namespace defNS = gcElem.getNamespace();

        RadarConfigEntry.GeoInfo gi = new RadarConfigEntry.GeoInfo();
        gi.northSouth = readGeospatialRange(gcElem.getChild("northsouth", defNS), CDM.LAT_UNITS);
        gi.eastWest = readGeospatialRange(gcElem.getChild("eastwest", defNS), CDM.LON_UNITS);
        gi.upDown = readGeospatialRange(gcElem.getChild("updown", defNS), "m");

        return gi;
    }

    protected static RadarConfigEntry.RangeInfo readGeospatialRange(Element spElem, String defUnits) {
        if (spElem == null) return null;

        Namespace defNS = spElem.getNamespace();
        double start = readDouble(spElem.getChild("start", defNS));
        double size = readDouble(spElem.getChild("size", defNS));

        String units = spElem.getChildText("units", defNS);
        if (units == null) units = defUnits;

        RadarConfigEntry.RangeInfo ri = new RadarConfigEntry.RangeInfo();
        ri.units = units;
        ri.start = start;
        ri.size = size;
        return ri;
    }

    protected static double readDouble(Element elem) {
        if (elem == null) return Double.NaN;
        String text = elem.getText();
        try {
            return Double.parseDouble(text);
        } catch (NumberFormatException e) {
            return Double.NaN;
        }
    }

    protected static DateRange readTimeCoverage(Element tElem) {
        if (tElem == null) return null;
        Namespace defNS = tElem.getNamespace();
        DateType start = readDate(tElem.getChild("start", defNS));
        DateType end = readDate(tElem.getChild("end", defNS));
        TimeDuration duration = readDuration(tElem.getChild("duration", defNS));
        TimeDuration resolution = readDuration(tElem.getChild("resolution", defNS));

        try {
            return new DateRange(start, end, duration, resolution);
        } catch (java.lang.IllegalArgumentException e) {
            return null;
        }
    }

    protected static DateType readDate(Element elem) {
        if (elem == null) return null;
        String format = elem.getAttributeValue("format");
        String type = elem.getAttributeValue("type");
        return makeDateType(elem.getText(), format, type);
    }

    protected static DateType makeDateType(String text, String format, String type) {
        if (text == null) return null;
        try {
            return new DateType(text, format, type);
        } catch (java.text.ParseException e) {
            return null;
        }
    }

    protected static TimeDuration readDuration(Element elem) {
        if (elem == null) return null;
        String text;
        try {
            text = elem.getText();
            return new TimeDuration(text);
        } catch (java.text.ParseException e) {
            return null;
        }
    }

    static public class RadarConfigEntry {
        public Path dataPath;
        public String name, urlPath, dataFormat, stationFile, doc;
        public String dateParseRegex, dateFmt, layout;
        public DateRange timeCoverage;
        public GeoInfo  spatialCoverage;
        public int crawlItems;
        public List<VarInfo> vars;

        static public class GeoInfo {
            RangeInfo eastWest, northSouth, upDown;
        }

        static public class RangeInfo {
            public String units;
            public double start, size;
        }

        static public class VarInfo {
            public String name, vocabName, units;
        }
    }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.nexrad2;

import ucar.nc2.util.TableParser;

import java.util.HashMap;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Map;
import java.io.IOException;
import java.io.InputStream;

import org.jdom2.*;
import org.jdom2.input.*;

/**
 * Manage Nexrad Stations "database"
 * @author caron
 */
public class NexradStationDB {

  private static boolean showStations = false;
  private static Map<String,Station> stationTableHash = null;
  private static Map<String,Station> stationTableHash1 = null;

  public static synchronized void init() throws IOException {
    if (stationTableHash == null)
      readStationTableXML();
  }

  public static Station get(String id) { return stationTableHash.get(id); }

  public static Station getByIdNumber(String idn) { return stationTableHash1.get(idn); }

  private static void readStationTableXML() throws IOException {
    stationTableHash = new HashMap<String,Station>();
    stationTableHash1 = new HashMap<String,Station>();
    ClassLoader cl = Level2VolumeScan.class.getClassLoader();
    InputStream is = cl.getResourceAsStream("resources/nj22/tables/nexradstns.xml");

    Document doc;
    SAXBuilder saxBuilder = new SAXBuilder();
    try {
      doc = saxBuilder.build(is);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }

    Element root = doc.getRootElement();
    List children = root.getChildren("station");
    for (int i = 0; i < children.size(); i++) {
      Element sElem =  (Element)children.get(i);
      String idn = sElem.getAttributeValue("idn");
      String id = sElem.getAttributeValue("id");
      String name = sElem.getAttributeValue("name");
      String st = sElem.getAttributeValue("st");
      String co = sElem.getAttributeValue("co");
      String lat = sElem.getAttributeValue("lat");
      String lon = sElem.getAttributeValue("lon");
      String elev = sElem.getAttributeValue("elev");

      Station s = new Station();
      s.id = "K"+id;
      s.name = name + "," + st+ "," + co;
      s.lat = parseDegree(lat);
      s.lon = parseDegree(lon);
      s.elev = Double.parseDouble(elev);

      stationTableHash.put(s.id, s);
      stationTableHash1.put(idn, s);
      if (showStations) System.out.println(" station= "+s);
    }
  }

    private static double parseDegree( String s) {
      StringTokenizer stoke = new StringTokenizer(s, ":");
      String degS = stoke.nextToken();
      String minS = stoke.nextToken();
      String secS = stoke.nextToken();

      try {
        double deg = Double.parseDouble( degS);
        double min = Double.parseDouble( minS);
        double sec = Double.parseDouble( secS);
        if (deg < 0)
          return deg - min/60 - sec/3600;
        else
          return deg + min/60 + sec/3600;
      } catch (NumberFormatException e) {
        e.printStackTrace();
      }
      return 0.0;
    }

  // this is the old Gempak table, not as precise
   private static void readStationTable() throws IOException {
    stationTableHash = new HashMap<String,Station>();

    ClassLoader cl = Level2VolumeScan.class.getClassLoader();
    InputStream is = cl.getResourceAsStream("resources/nj22/tables/nexrad.tbl");

    List<TableParser.Record> recs = TableParser.readTable(is, "3,15,46, 54,60d,67d,73d", 50000);
    for (TableParser.Record record : recs) {
      Station s = new Station();
      s.id = "K" + record.get(0);
      s.name = record.get(2) + " " + record.get(3);
      s.lat = (Double) record.get(4) * .01;
      s.lon = (Double) record.get(5) * .01;
      s.elev = (Double) record.get(6);

      stationTableHash.put(s.id, s);
      if (showStations) System.out.println(" station= " + s);
    }
  }

  public static class Station {
    public String id, name;
    public double lat;
    public double lon;
    public double elev;

    public String toString() { return id +" <"+ name +">   "+ lat +" "+ lon +" "+ elev; }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.noaa;

import com.google.protobuf.InvalidProtocolBufferException;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.AxisType;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.CF;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.stream.NcStream;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.TableParser;
import ucar.unidata.io.RandomAccessFile;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Nomads GLOBAL HISTORICAL CLIMATOLOGY NETWORK MONTHLY (GHCNM) v3 Beta.
 * Ascii file. Write index into .ncsx using protobug (experimental)
 * IOSP can then recognize the ncsx file, so it can be passed into NetdfFile.open() instead of the ascii file.
 * Otherwise you have to explicitly specify the iosp, eg:
 * <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" location="filname" iosp="ucar.nc2.iosp.noaa.Ghcnm" />
 *

 * The index has list of stations and offsets into station file and data file.
 * since the data file has one station's data grouped together, this efficiently answers "get all data for station".
 * One could sort datafile by time, and add time index, to answer "get all data for time range".
 *

 *

 * Protobuf generation:
 * cd c:/dev/tds4.2/thredds/cdm/src/main/java
 * protoc --proto_path=. --java_out=. ucar/nc2/iosp/noaa/GhcnmIndex.proto
 *

 *
 * LOOK probable file leaks
 *
 * @author caron
 * @since Dec 8, 2010
 */
public class Ghcnm extends AbstractIOServiceProvider {

  /*
  ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/v3/README

    2.1 METADATA

       The metadata has been carried over from GHCN-Monthly v2.  This would
       include basic geographical station information such as latitude,
       longitude, elevation, station name, etc., and also extended metadata
       information, such as surrounding vegetation, etc.

    2.1.1 METADATA FORMAT

       Variable          Columns      Type
       --------          -------      ----

       ID                 1-11        Integer
       LATITUDE          13-20        Real
       LONGITUDE         22-30        Real
       STNELEV           32-37        Real
       NAME              39-68        Character
       GRELEV            70-73        Integer
       POPCLS            74-74        Character
       POPSIZ            76-79        Integer
       TOPO              80-81        Character
       STVEG             82-83        Character
       STLOC             84-85        Character
       OCNDIS            86-87        Integer
       AIRSTN            88-88        Character
       TOWNDIS           89-90        Integer
       GRVEG             91-106       Character
       POPCSS            107-107      Character

       Variable Definitions:

       ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
           the WMO stnId if the station is a WMO station.  It is a WMO station if
           digits 9-11="000".

       LATITUDE: latitude of station in decimal degrees

       LONGITUDE: longitude of station in decimal degrees

       STELEV: is the station elevation in meters. -999.0 = missing.

       NAME: station name

       GRELEV: station elevation in meters estimated from gridded digital
               terrain data

       POPCLS: population class
               (U=Urban (>50,000 persons);
               (S=Suburban (>=10,000 and <= 50,000 persons);
               (R=Rural (<10,000 persons)
               City and town boundaries are determined from location of station
               on Operational Navigation Charts with a scale of 1 to 1,000,000.
               For cities > 100,000 persons, population data were provided by
               the United Nations Demographic Yearbook. For smaller cities and
               towns several atlases were uses to determine population.

       POPSIZ: the population of the city or town the station is location in
               (expressed in thousands of persons).

       TOPO: type of topography in the environment surrounding the station,
             (Flat-FL,Hilly-HI,Mountain Top-MT,Mountainous Valley-MV).

       STVEG: type of vegetation in environment of station if station is Rural
              and when it is indicated on the Operational Navigation Chart
              (Desert-DE,Forested-FO,Ice-IC,Marsh-MA).

       STLOC: indicates whether station is near lake or ocean (<= 30 km of
              ocean-CO, adjacent to a lake at least 25 square km-LA).

       OCNDIS: distance to nearest ocean/lake from station (km).

       AIRSTN: airport station indicator (A=station at an airport).

       TOWNDIS: distance from airport to center of associated city or town (km).

       GRVEG: vegetation type at nearest 0.5 deg x 0.5 deg gridded data point of
              vegetation dataset (44 total classifications).

              BOGS, BOG WOODS
              COASTAL EDGES
              COLD IRRIGATED
              COOL CONIFER
              COOL CROPS
              COOL DESERT
              COOL FIELD/WOODS
              COOL FOR./FIELD
              COOL GRASS/SHRUB
              COOL IRRIGATED
              COOL MIXED
              EQ. EVERGREEN
              E. SOUTH. TAIGA
              HEATHS, MOORS
              HIGHLAND SHRUB
              HOT DESERT
              ICE
              LOW SCRUB
              MAIN TAIGA
              MARSH, SWAMP
              MED. GRAZING
              NORTH. TAIGA
              PADDYLANDS
              POLAR DESERT
              SAND DESERT
              SEMIARID WOODS
              SIBERIAN PARKS
              SOUTH. TAIGA
              SUCCULENT THORNS
              TROPICAL DRY FOR
              TROP. MONTANE
              TROP. SAVANNA
              TROP. SEASONAL
              TUNDRA
              WARM CONIFER
              WARM CROPS
              WARM DECIDUOUS
              WARM FIELD WOODS
              WARM FOR./FIELD
              WARM GRASS/SHRUB
              WARM IRRIGATED
              WARM MIXED
              WATER
              WOODED TUNDRA

       POPCSS: population class as determined by Satellite night lights
               (C=Urban, B=Suburban, A=Rural)

    2.2  DATA

         The data within GHCNM v3 beta for the time being consist of monthly
         average temperature, for the 7280 stations contained within GHCNM v2.
         Several new sources have been added to v3 beta, and a new "3 flag"
         format has been introduced, similar to that used within the Global
         Historical Climatology Network-Daily (GHCND).

    2.2.1 DATA FORMAT

          Variable          Columns      Type
          --------          -------      ----

          ID                 1-11        Integer
          YEAR              12-15        Integer
          ELEMENT           16-19        Character
          VALUE1            20-24        Integer
          DMFLAG1           25-25        Character
          QCFLAG1           26-26        Character
          DSFLAG1           27-27        Character
            .                 .             .
            .                 .             .
            .                 .             .
          VALUE12          108-112       Integer
          DMFLAG12         113-113       Character
          QCFLAG12         114-114       Character
          DSFLAG12         115-115       Character

          Variable Definitions:

          ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
              the WMO stnId if the station is a WMO station.  It is a WMO station if
              digits 9-11="000".

          YEAR: 4 digit year of the station record.

          ELEMENT: element type, currently just "TAVG".

          VALUE: monthly value (MISSING=-9999)

          DMFLAG: data measurement flag, nine possible values:

                  blank = no measurement information applicable
                  a-i = number of days missing in calculation of monthly mean
                        temperature (currently only applies to the 1218 USHCN
                        V2 stations included within GHCNM)

          QCFLAG: quality control flag, seven possibilities within
                  quality controlled unadjusted (qcu) dataset, and 2
                  possibilities within the quality controlled adjusted (qca)
                  dataset.

                  Quality Controlled Unadjusted (QCU) QC Flags:

                  BLANK = no failure of quality control check or could not be
                          evaluated.

                  D = monthly value is part of an annual series of values that
                      are exactly the same (e.fldno. duplicated) within another
                      year in the station's record.

                  K = monthly value is part of a consecutive run (e.fldno. streak)
                      of values that are identical.  The streak must be >= 4
                      months of the same value.

                  L = monthly value is isolated in time within the station
                      record, and this is defined by having no immediate non-
                      missing values 18 months on either side of the value.

                  O = monthly value that is >= 5 bi-weight standard deviations
                      from the bi-weight mean.  Bi-weight statistics are
                      calculated from a series of all non-missing values in
                      the station's record for that particular month.

                  S = monthly value has failed spatial consistency check
                      (relative to their respective climatological means to
                       concurrent z-scores at the nearest 20 neighbors located
                       withing 500 km of the target; A temperature fails if
                       (i) its z-score differs from the regional (target and
                       neighbor) mean z-score by at least 3.5 standard
                       deviations and (ii) the target's temperature anomaly
                       differs by at least 2.5 deg C from all concurrent
                       temperature anomalies at the neighbors.

                  T = monthly value has failed temporal consistency check
                      (temperatures whose anomalies differ by more than
                      4 deg C from concurent anomalies at the five nearest
                      neighboring stations whose temperature anomalies are
                      well correlated with the target (correlation > 0.7 for
                      the corresponding calendar monthly).

                  W = monthly value is duplicated from the previous month,
                      based upon regional and spatial criteria and is only
                      applied from the year 2000 to the present.

                  Quality Controlled Adjusted (QCA) QC Flags:

                  M = values with a non-blank quality control flag in the "qcu"
                      dataset are set to missing the adjusted dataset and given
                      an "M" quality control flag.

                  X = pairwise algorithm removed the value because of too many
                      inhomogeneities.

          DSFLAG: data source flag for monthly value, 18 possibilities:

                  C = Monthly Climatic Data of the World (MCDW) QC completed
                      but value is not yet published

                  G = GHCNM v2 station, that was not a v2 station that had multiple
                      time series (for the same element).

                  K = received by the UK Met Office

                  M = Final (Published) Monthly Climatic Data of the World
                     (MCDW)

                  N = Netherlands, KNMI (Royal Netherlans Meteorological
                      Institute)

                  P = CLIMAT (Data transmitted over the GTS, not yet fully
                      processed for the MCDW)

                  U = USHCN v2

                  W = World Weather Records (WWR), 9th series 1991 through 2000

             0 to 9 = For any station originating from GHCNM v2 that had
                      multiple time series for the same element, this flag
                      represents the 12th digit in the ID from GHCNM v2.
                      See section 2.2.2 for additional information.

    2.2.2 STATIONS WITH MULTIPLE TIME SERIES

          The GHCNM v2 contained several thousand stations that had multiple
          time series of monthly mean temperature data.  The 12th digit of
          each data record, indicated the time series number, and thus there
          was a potential maximum of 10 time series (e.fldno. 0 through 9).  These
          same stations in v3 beta have undergone a merge process, to reduce
          the station time series to one single series, based upon these
          original and at most 10 time series.

          A simple algorithm was applied to perform the merge.  The algorithm
          consisted of first finding the length (based upon number of non
          missing observations) for each of the time series and then
          combining all of the series into one based upon a priority scheme
          that would "write" data to the series for the longest series last.

          Therefore, if station A, had 3 time series of TAVG data, as follows:

          1900 to 1978 (79 years of data) [series 1]
          1950 to 1985 (36 years of data) [series 2]
          1990 to 2007 (18 years of data) [series 3]

          The final series would consist of:

          1900 to 1978 [series 1]
          1979 to 1985 [series 2]
          1990 to 2007 [series 3]

          The original series number in GHCNM v2, is retained in the GHCNM v3
          beta data source flag.

          One caveat to this merge process, is that in the final GHCNM v3 beta
          processing there is still a master level construction process
          performed daily, where the entire dataset is construction according
          to a source order overwrite hiearchy (section 2.3), and it is
          possible that higher order data sources may be interspersed within
          the 3 series listed above.

    2.3 DATA SOURCE HIERARCHY

        The GHCNM v3 beta is reprocessed on a daily basis, which means as a
        part of that reprocessing, the dataset is reconstructed from all
        original sources. The advantage to this process is when source
        datasets are corrected and/or updated the inclusion into GHCNM v3
        beta is seemless.  The following sources (more fully described in
        section 2.2.1) have the following overwrite precedance within the
        daily reprocessing of GHCNM v3 (e.fldno. source K overwrites source P)

        P,K,G,U,0-9,C,N,M,W
  */

  /*
  data
          1         2         3         4         5         6         7         8         9         10        11        12
0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
101603550001878TAVG  890  1  950  1 1110  1 1610  1 1980  1 2240  1 2490  1 2680  1 2320  1 2057E   1370  1 1150  1
101603550001932TAVG 1010  1  980  1-9999    1420  1 1840  1-9999    2290  1-9999    2440  1-9999   -9999   -9999

 matches = true
 group 1 == 101603550001932
 group 2 ==  1010
 group 3 ==   1
 group 4 ==   980
 group 5 ==   1
 group 6 == -9999
 group 7 ==
 group 8 ==  1420
 group 9 ==   1
 group 10 ==  1840
 group 11 ==   1
 group 12 == -9999
 group 13 ==
 group 14 ==  2290
 group 15 ==   1
 group 16 == -9999
 group 17 ==
 group 18 ==  2440
 group 19 ==   1
 group 20 == -9999
 group 21 ==
 group 22 == -9999
 group 23 ==
 group 24 == -9999
 group 25 == null

 see testRegexp.testGhcnm()
*/
  private static final String p = "(\\d{11})(\\d{4})TAVG([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)([ \\-\\d]{5})(...)?.*";
  private static final Pattern dataPattern = Pattern.compile(p);

  private static final String RECORD = "all_data";
  private static final String DIM_NAME = "month";
  private static final String STNID = "stnid";

  private static final String TIME = "time";
  private static final String YEAR = "year";
  private static final String VALUE = "value";
  private static final String DMFLAG = "dm";
  private static final String QCFLAG = "qc";
  private static final String DSFLAG = "ds";

  private static final String STNS = "station";
  private static final String LAT = "lat";
  private static final String LON = "lon";
  private static final String STELEV = "elevation";
  private static final String NAME = "name";
  private static final String GRELEV = "grelev";
  private static final String POPCLS = "popClass";
  private static final String POPSIZ = "popSize";
  private static final String TOPO = "topoType";
  private static final String STVEG = "vegType";
  private static final String STLOC = "ocean";
  private static final String OCNDIS = "oceanDist";
  private static final String AIRSTN = "airportId";
  private static final String TOWNDIS = "townDist";
  private static final String GRVEG = "vegType";
  private static final String POPCSS = "popClassFromLights";
  private static final String WMO = "wmoId";

  private static final String STN_DATA = "stn_data";

  private NetcdfFile ncfile;
  private RandomAccessFile stnRaf;

  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    String dataFile = raf.getLocation();
    int pos = dataFile.lastIndexOf(".");
    if (pos <= 0) return false;
    String base = dataFile.substring(0, pos);
    String ext = dataFile.substring(pos);

    // must be data file or index file
    if (!ext.equals(DAT_EXT) && !ext.equals(IDX_EXT))
      return false;

    if (ext.equals(IDX_EXT)) {
      // data, stn files must be in the same directory
      File datFile = new File(base + DAT_EXT);
      if (!datFile.exists())
        return false;
      File stnFile = new File(base + STN_EXT);
      if (!stnFile.exists())
        return false;

      raf.seek(0);
      byte[] b = new byte[MAGIC_START.length()];
      raf.readFully(b);
      String test = new String(b, CDM.utf8Charset);
      return test.equals(MAGIC_START);

    } else {
      // stn files must be in the same directory
      File stnFile = new File(base + STN_EXT);
      return (stnFile.exists()); // LOOK BAD!! NOT GOOD ENOUGH
    }
  }

  @Override
  public void close() throws java.io.IOException {
    if (raf != null)
      raf.close();
    if (stnRaf != null)
      stnRaf.close();

    raf = null;
    stnRaf = null;
  }

  @Override
  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    super.open(raf, ncfile, cancelTask);
    this.ncfile = ncfile;

    String dataFile = raf.getLocation();
    int pos = dataFile.lastIndexOf(".");
    String base = dataFile.substring(0, pos);
    String ext = dataFile.substring(pos);

    // did the index file get passed in ?
    if (ext.equals(IDX_EXT)) {
      // must be in the same directory
      File datFile = new File(base + DAT_EXT);
      if (!datFile.exists())
        throw new FileNotFoundException(datFile.getPath() + " must exist");
      File stnFile = new File(base + STN_EXT);
      if (!stnFile.exists())
        throw new FileNotFoundException(stnFile.getPath() + " must exist");

      this.raf = RandomAccessFile.acquire(base + DAT_EXT);
      this.stnRaf = RandomAccessFile.acquire(base + STN_EXT);
      readIndex(raf.getLocation());
      raf.close();

    } else { // must be the data file, we will write an index if needed below
      this.raf = raf;
      if (!(ext.equals(DAT_EXT)))
        throw new FileNotFoundException("Ghcnm: file must end with " + DAT_EXT);

      File stnFile = new File(base + STN_EXT);
      if (!stnFile.exists())
        throw new FileNotFoundException(stnFile.getPath() + " must exist");
      this.stnRaf = RandomAccessFile.acquire(base + STN_EXT);
    }

    //////////////////////////////////////////////////
    // LOOK - can we externalize config ??

    /*
          ID                 1-11        Integer
          YEAR              12-15        Integer
          ELEMENT           16-19        Character
          VALUE1            20-24        Integer
          DMFLAG1           25-25        Character
          QCFLAG1           26-26        Character
          DSFLAG1           27-27        Character

          ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
              the WMO stnId if the station is a WMO station.  It is a WMO station if
              digits 9-11="000".
     */

    TableParser dataParser = new TableParser("11L,15i,19,24i,25,26,27");
    Structure dataSeq = new Sequence(ncfile, null, null, RECORD);
    ncfile.addVariable(null, dataSeq);
    ncfile.addDimension(null, new Dimension(DIM_NAME, 12));
    Variable v;

    makeMember(dataSeq, STNID, DataType.LONG, null, "station stnId", null, null, null);
    makeMember(dataSeq, YEAR, DataType.INT, null, "year of the station record", null, null, null);
    v = makeMember(dataSeq, VALUE, DataType.FLOAT, DIM_NAME, "monthly mean temperature", "Celsius", null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9999));
    dataParser.getField(3).setScale(.01f);
    makeMember(dataSeq, DMFLAG, DataType.CHAR, DIM_NAME, "data management flag", null, null, null);
    makeMember(dataSeq, QCFLAG, DataType.CHAR, DIM_NAME, "quality control flag", null, null, null);
    makeMember(dataSeq, DSFLAG, DataType.CHAR, DIM_NAME, "data source flag", null, null, null);

    // synthetic time variable
    v = makeMember(dataSeq, TIME, DataType.STRING, null, "starting time of the record", null, null, AxisType.Time);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, "missing"));

    StructureMembers dataSm = dataSeq.makeStructureMembers();
    dataSm.findMember(STNID).setDataObject(dataParser.getField(0));
    dataSm.findMember(YEAR).setDataObject(dataParser.getField(1));
    dataSm.findMember(VALUE).setDataObject(dataParser.getField(3));
    dataSm.findMember(DMFLAG).setDataObject(dataParser.getField(4));
    dataSm.findMember(QCFLAG).setDataObject(dataParser.getField(5));
    dataSm.findMember(DSFLAG).setDataObject(dataParser.getField(6));
    dataSeq.setSPobject(new Vinfo(this.raf, dataSm));
    stnIdFromData = dataParser.getField(0);

    // synthetic time variable
    TableParser.Field org = dataParser.getField(1); // YEAR
    TableParser.Field derived = dataParser.addDerivedField(org, new TableParser.Transform() {
      public Object derive(Object org) {
        int year = (Integer) org;
        return year + "-01-01T00:00:00Z";
      }
    }, String.class);
    dataSm.findMember(TIME).setDataObject(derived);

    /*
       ID                 1-11        Integer
       LATITUDE          13-20        Real
       LONGITUDE         22-30        Real
       STNELEV           32-37        Real
       NAME              39-68        Character
       GRELEV            70-73        Integer
       POPCLS            74-74        Character
       POPSIZ            76-79        Integer
       TOPO              80-81        Character
       STVEG             82-83        Character
       STLOC             84-85        Character
       OCNDIS            86-87        Integer
       AIRSTN            88-88        Character
       TOWNDIS           89-90        Integer
       GRVEG             91-106       Character
       POPCSS            107-107      Character
      */

    TableParser stnParser = new TableParser("11L,20d,30d,37d,68,73i,74,79i,81,83,85,87i,88,90i,106,107");
    Structure stnSeq = new Sequence(ncfile, null, null, STNS);
    ncfile.addVariable(null, stnSeq);

    v = makeMember(stnSeq, STNID, DataType.LONG, null, "station id", null, null, null);
    v.addAttribute(new Attribute(CF.CF_ROLE, CF.TIMESERIES_ID));
    makeMember(stnSeq, LAT, DataType.FLOAT, null, "latitude", CDM.LAT_UNITS, null, null);
    makeMember(stnSeq, LON, DataType.FLOAT, null, "longitude", CDM.LON_UNITS, null, null);
    makeMember(stnSeq, STELEV, DataType.FLOAT, null, "elevation", "m", null, null);
    v = makeMember(stnSeq, NAME, DataType.STRING, null, "station name", null, null, null);
    v.addAttribute(new Attribute(CF.STANDARD_NAME, CF.STATION_DESC));
    makeMember(stnSeq, GRELEV, DataType.INT, null, "elevation estimated from gridded digital terrain data", "m", null, null);
    makeMember(stnSeq, POPCLS, DataType.CHAR, null, "population class", null, null, null);
    v = makeMember(stnSeq, POPSIZ, DataType.INT, null, "population of the city or town the station is located in", "thousands of persons", null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9));
    makeMember(stnSeq, TOPO, DataType.STRING, null, "type of topography in the environment surrounding the station", null, null, null);
    makeMember(stnSeq, STVEG, DataType.STRING, null, "type of vegetation in environment of station", null, null, null);
    makeMember(stnSeq, STLOC, DataType.STRING, null, "station is near lake or ocean", null, null, null);
    v = makeMember(stnSeq, OCNDIS, DataType.INT, null, "distance to nearest ocean/lake", "km", null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9));
    makeMember(stnSeq, AIRSTN, DataType.CHAR, null, "airport station indicator", null, null, null);
    v = makeMember(stnSeq, TOWNDIS, DataType.INT, null, "distance from airport to center of associated city or town", "km", null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9));
    makeMember(stnSeq, GRVEG, DataType.STRING, null, "vegetation type at nearest 0.5 deg x 0.5 deg gridded data point of vegetation dataset", null, null, null);
    makeMember(stnSeq, POPCSS, DataType.CHAR, null, "population class as determined by satellite night lights", null, null, null);

    // nested sequence - just the data for this station
    Structure nestedSeq = new Sequence(ncfile, null, stnSeq, STN_DATA);
    stnSeq.addMemberVariable(nestedSeq);

    v = makeMember(nestedSeq, YEAR, DataType.INT, null, "year", null, null, null);
    v.addAttribute(new Attribute(CDM.UNITS, "years since 0000-01-01T00:00"));
    v = makeMember(nestedSeq, VALUE, DataType.FLOAT, DIM_NAME, "monthly mean temperature", "Celsius", null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9999));
    dataParser.getField(3).setScale(.01f);
    makeMember(nestedSeq, DMFLAG, DataType.CHAR, DIM_NAME, "data management flag", null, null, null);
    makeMember(nestedSeq, QCFLAG, DataType.CHAR, DIM_NAME, "quality control flag", null, null, null);
    makeMember(nestedSeq, DSFLAG, DataType.CHAR, DIM_NAME, "data source flag", null, null, null);

    // synthetic time variable in nested seq
    v = makeMember(nestedSeq, TIME, DataType.STRING, null, "starting time of the record", null, null, AxisType.Time);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, "missing"));

    StructureMembers nestedSm = nestedSeq.makeStructureMembers();
    nestedSm.findMember(YEAR).setDataObject(dataParser.getField(1));
    nestedSm.findMember(VALUE).setDataObject(dataParser.getField(3));
    nestedSm.findMember(DMFLAG).setDataObject(dataParser.getField(4));
    nestedSm.findMember(QCFLAG).setDataObject(dataParser.getField(5));
    nestedSm.findMember(DSFLAG).setDataObject(dataParser.getField(6));
    nestedSeq.setSPobject(new Vinfo(this.raf, nestedSm));
    stnDataMembers = nestedSm;

    // synthetic time variable in nested seq
    TableParser.Field org2 = dataParser.getField(1); // YEAR
    TableParser.Field derived2 = dataParser.addDerivedField(org2, new TableParser.Transform() {
      public Object derive(Object org) {
        int year = (Integer) org;
        return year + "-01-01T00:00:00Z";
      }
    }, String.class);
    nestedSm.findMember(TIME).setDataObject(derived2);

    //// finish the stn sequence

    // synthetic wmo variable in station
    v = makeMember(stnSeq, WMO, DataType.INT, null, "WMO station id", null, null, null);
    v.addAttribute(new Attribute(CDM.MISSING_VALUE, -9999));
    v.addAttribute(new Attribute(CF.STANDARD_NAME, CF.STATION_WMOID));

    StructureMembers stnSm = stnSeq.makeStructureMembers();
    int count = 0;
    int n = stnParser.getNumberOfFields();
    for (StructureMembers.Member m : stnSm.getMembers()) {
      if (count < n)
        m.setDataObject(stnParser.getField(count++));
    }
    stnSeq.setSPobject(new Vinfo(stnRaf, stnSm));

    // synthetic wmo variable in station
    /* ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
           the WMO id if the station is a WMO station.  It is a WMO station if
           digits 9-11="000". */
    TableParser.Field org3 = stnParser.getField(0); // STNID
    TableParser.Field derived3 = stnParser.addDerivedField(org3, new TableParser.Transform() {
      public Object derive(Object org) {
        long stnid = (Long) org;
        return (stnid % 1000 == 0) ? new Integer((int) (stnid / 1000) % 100000) : new Integer(-9999);
      }
    }, int.class);
    stnSm.findMember(WMO).setDataObject(derived3);

    //// global attribites
    ncfile.addAttribute(null, new Attribute("title", "Version 3 of the GHCN-Monthly dataset of land surface mean temperatures"));
    ncfile.addAttribute(null, new Attribute(CDM.CONVENTIONS, "CDM"));
    ncfile.addAttribute(null, new Attribute("CF:featureType", "timeSeries"));
    ncfile.addAttribute(null, new Attribute("see", "http://www.ncdc.noaa.gov/ghcnm, ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/v3"));
    ncfile.finish();

    // make index file if needed
    File idxFile = new File(base + IDX_EXT);
    if (!idxFile.exists())
      makeIndex(idxFile);
    else
      readIndex(idxFile.getPath());
  }

  private Variable makeMember(Structure s, String shortName, DataType dataType, String dims, String longName, String units, String cfName,
                              AxisType atype) {

    Variable v = new Variable(ncfile, null, s, shortName, dataType, dims);
    v.addAttribute(new Attribute(CDM.LONG_NAME, longName));
    if (cfName != null)
      v.addAttribute(new Attribute(CF.STANDARD_NAME, cfName));

    if (units != null)
      v.addAttribute(new Attribute(CDM.UNITS, units));

    if (atype != null)
      v.addAttribute(new Attribute(_Coordinate.AxisType, atype.toString()));

    s.addMemberVariable(v);

    return v;
  }

  private static class Vinfo {
    RandomAccessFile raf;
    StructureMembers sm;
    int nelems = -1;

    private Vinfo(RandomAccessFile raf, StructureMembers sm) {
      this.sm = sm;
      this.raf = raf;
    }
  }

  ////////////////////////////////////////////////////////////////////////////////////////////

  private static final String STN_EXT = ".inv";
  private static final String DAT_EXT = ".dat";
  private static final String IDX_EXT = ".ncsx";
  private static final String MAGIC_START = "GhncmIndex";
  private static final int version = 1;

  private HashMap<Long, StationIndex> map = new HashMap<Long, StationIndex>(10000);
  private TableParser.Field stnIdFromData;
  private StructureMembers stnDataMembers;

  private void readIndex(String indexFilename) throws IOException {
    try (FileInputStream fin = new FileInputStream(indexFilename)) {
      if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(CDM.utf8Charset)))
        throw new IllegalStateException("bad index file");
      int version = fin.read();
      if (version != 1)
        throw new IllegalStateException("Bad version = " + version);

      int count = NcStream.readVInt(fin);

      for (int i = 0; i < count; i++) {
        int size = NcStream.readVInt(fin);
        byte[] pb = new byte[size];
        NcStream.readFully(fin, pb);
        StationIndex si = decodeStationIndex(pb);
        map.put(si.stnId, si);
      }
    }

    System.out.println(" read index map size=" + map.values().size());
  }

  private void makeIndex(File indexFile) throws IOException {
    // get map of Stations
    Sequence stnSeq = (Sequence) ncfile.findVariable(STNS);
    Vinfo stnInfo = (Vinfo) stnSeq.getSPobject();
    StructureMembers.Member m = stnInfo.sm.findMember(STNID);
    TableParser.Field f = (TableParser.Field) m.getDataObject();
    int stnCount = 0;

    // read through entire file
    stnInfo.raf.seek(0);
    while (true) {
      long stnPos = stnInfo.raf.getFilePointer();
      String line = stnInfo.raf.readLine();
      if (line == null) break;
      StationIndex s = new StationIndex();
      Long id = (Long) f.parse(line);
      map.put(id, s);
      s.stnId = id;
      s.stnPos = stnPos;
      stnCount++;
    }

    // assumes that the stn data is in order by stnId
    Sequence dataSeq = (Sequence) ncfile.findVariable(RECORD);
    Vinfo dataInfo = (Vinfo) dataSeq.getSPobject();
    m = dataInfo.sm.findMember(STNID);
    f = (TableParser.Field) m.getDataObject();
    StationIndex currStn = null;
    int totalCount = 0;

    // read through entire file
    dataInfo.raf.seek(0);
    while (true) {
      long dataPos = dataInfo.raf.getFilePointer();
      String line = dataInfo.raf.readLine();
      if (line == null) break;

      Long id = (Long) f.parse(line);
      if ((currStn == null) || (currStn.stnId != id)) {
        StationIndex s = map.get(id);
        if (s == null)
          System.out.printf("Cant find %d%n", id);
        else if (s.dataCount != 0)
          System.out.printf("Not in order %d at pos %d %n", id, dataPos);
        else {
          s.dataPos = dataPos;
          totalCount++;
        }
        currStn = s;
      }
      if (currStn != null) currStn.dataCount++;
    }
    //System.out.printf("ok stns=%s data=%d%n", stnCount, totalCount);

    //////////////////////////////
    // write the index file
    try (FileOutputStream fout = new FileOutputStream(indexFile)) { // LOOK need DiskCache for non-writeable directories
      long size = 0;

      //// header message
      fout.write(MAGIC_START.getBytes(CDM.utf8Charset));
      fout.write(version);
      size += NcStream.writeVInt(fout, stnCount);

    /* byte[] pb = encodeStationListProto( map.values());
    size += NcStream.writeVInt(fout, pb.length);
    size += pb.length;
    fout.write(pb); */

      for (StationIndex s : map.values()) {
        byte[] pb = s.encodeStationProto();
        size += NcStream.writeVInt(fout, pb.length);
        size += pb.length;
        fout.write(pb);
      }
    }

    //System.out.println(" index size=" + size);
  }

  private StationIndex decodeStationIndex(byte[] data) throws InvalidProtocolBufferException {
    ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto = GhcnmProto.StationIndex.parseFrom(data);
    return new StationIndex(proto);
  }

  private static class StationIndex {
    long stnId;
    long stnPos; // file pos in inv file
    long dataPos; // file pos of first data line in the data file
    int dataCount; // number of data records

    StationIndex() {
    }

    StationIndex(ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto) {
      this.stnId = proto.getStnid();
      this.stnPos = proto.getStnPos();
      this.dataPos = proto.getDataPos();
      this.dataCount = proto.getDataCount();
    }

    private byte[] encodeStationProto() {
      GhcnmProto.StationIndex.Builder builder = GhcnmProto.StationIndex.newBuilder();
      builder.setStnid(stnId);
      builder.setStnPos(stnPos);
      builder.setDataPos(dataPos);
      builder.setDataCount(dataCount);
      ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto = builder.build();
      return proto.toByteArray();
    }
  }

  /* private byte[] encodeStationListProto(Collection<StationIndex> stns)  {
    GhcnmProto.StationIndexList.Builder listBuilder = GhcnmProto.StationIndexList.newBuilder();
    for (StationIndex s : stns) {
      GhcnmProto.StationIndex.Builder builder = GhcnmProto.StationIndex.newBuilder();
      builder.setStnid(s.stnId);
      builder.setStnPos(s.stnPos);
      builder.setDataPos(s.dataPos);
      builder.setDataCount(s.dataCount);
      listBuilder.addList(builder);
    }
    return listBuilder.build().toByteArray();
  } */

  ////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Get a unique stnId for this file type.
   *
   * @return registered stnId of the file type
   * @see "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
   */
  public String getFileTypeId() {
    return "GHCNM";
  }

  /**
   * Get a human-readable description for this file type.
   *
   * @return description of the file type
   * @see "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
   */
  public String getFileTypeDescription() {
    return "GLOBAL HISTORICAL CLIMATOLOGY NETWORK MONTHLY";
  }

  /**
   * Returns an ArraySequence, no subsetting is allowed.
   *
   * @param v2      a top-level Variable
   * @param section the section of data to read.
   *                There must be a Range for each Dimension in the variable, in order.
   *                Note: no nulls allowed. IOSP may not modify.
   * @return ArraySequence
   * @throws IOException
   * @throws InvalidRangeException
   */
  public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
    Vinfo vinfo = (Vinfo) v2.getSPobject();
    return new ArraySequence(vinfo.sm, new SeqIter(vinfo), vinfo.nelems);
  }

  /**
   * Get the structure iterator
   *
   * @param s          the Structure
   * @param bufferSize the buffersize
   * @return the data iterator
   * @throws java.io.IOException if problem reading data
   */
  public StructureDataIterator getStructureIterator(Structure s, int bufferSize) throws java.io.IOException {
    Vinfo vinfo = (Vinfo) s.getSPobject();
    return new SeqIter(vinfo);
  }

  private class SeqIter implements StructureDataIterator {
    private Vinfo vinfo;
    private long bytesRead;
    private long totalBytes;
    private int recno;
    private StructureData curr;

    SeqIter(Vinfo vinfo) throws IOException {
      this.vinfo = vinfo;
      totalBytes = (int) raf.length();
      vinfo.raf.seek(0);
    }

    @Override
    public StructureDataIterator reset() {
      bytesRead = 0;
      recno = 0;

      try {
        vinfo.raf.seek(0);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return this;
    }

    @Override
    public boolean hasNext() throws IOException {
      boolean more = (bytesRead < totalBytes); // && (recno < 10);
      if (!more) {
        vinfo.nelems = recno;
        //System.out.printf("nelems=%d%n", recno);
        return false;
      }
      curr = reallyNext();
      more = (curr != null);
      if (!more) {
        vinfo.nelems = recno;
        //System.out.printf("nelems=%d%n", recno);
        return false;
      }
      return more;
    }

    @Override
    public StructureData next() throws IOException {
      return curr;
    }

    private StructureData reallyNext() throws IOException {
      String line;
      while (true) {
        line = vinfo.raf.readLine();
        if (line == null) return null;
        if (line.startsWith("#")) continue;
        if (line.trim().length() == 0) continue;
        break;
      }
      //System.out.printf("%s%n", line);
      bytesRead = vinfo.raf.getFilePointer();
      recno++;
      return new StructureDataAsciiGhcnm(vinfo.sm, line);
    }

    @Override
    public int getCurrentRecno() {
      return recno - 1;
    }

  }

  private class StnDataIter implements StructureDataIterator {
    private StructureMembers sm;
    private int countRead;
    private StationIndex stationIndex;

    StnDataIter(StructureMembers sm, StationIndex stationIndex) {
      this.sm = sm;
      this.stationIndex = stationIndex;
      reset();
    }

    @Override
    public StructureDataIterator reset() {
      countRead = 0;
      try {
        raf.seek(stationIndex.dataPos);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return this;
    }

    @Override
    public boolean hasNext() throws IOException {
      return (countRead < stationIndex.dataCount);
    }

    @Override
    public StructureData next() throws IOException {
      String line;
      while (true) {
        line = raf.readLine();
        if (line == null) return null;
        if (line.startsWith("#")) continue;
        if (line.trim().length() == 0) continue;
        break;
      }
      //System.out.printf("%s%n", line);
      countRead++;
      return new StructureDataAscii(sm, line);
    }

    @Override
    public int getCurrentRecno() {
      return countRead - 1;
    }

  }

  private class StructureDataAsciiGhcnm extends StructureDataAscii {
    StructureDataAsciiGhcnm(StructureMembers members, String line) {
      super(members, line);
    }

    @Override
    public ArraySequence getArraySequence(StructureMembers.Member m) {
      Long stnId = (Long) stnIdFromData.parse(line); // extract the station id
      StationIndex si = map.get(stnId); // find its index
      return new ArraySequence(stnDataMembers, new StnDataIter(stnDataMembers, si), -1);
    }
  }

  ////////////////////////////////////

  static private int parseLine(String line) throws IOException {
    int balony = 0;
    Matcher matcher = dataPattern.matcher(line);
    if (matcher.matches()) {
      for (int i = 1; i <= matcher.groupCount(); i++) {
        String r = matcher.group(i);
        if (r == null) continue;
        int value = (int) Long.parseLong(r.trim());
        balony += value;
      }
    } else {
      System.out.printf("Fail on %s%n", line);
    }
    return balony;
  }

  static private void readDataRegexp(String filename) throws IOException {
    int balony = 0;
    long start = System.currentTimeMillis();
    System.out.printf("regexp %s%n", filename);
    try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {
      String line;
      while (true) {
        line = raf.readLine();
        if (line == null) break;
        if (line.startsWith("#")) continue;
        if (line.trim().length() == 0) continue;
        balony += parseLine(line);
      }
    }

    long took = System.currentTimeMillis() - start;
    System.out.printf("DONE %d == %d msecs%n", balony, took);
  }

  static private void readData(String filename) throws IOException {
    long start = System.currentTimeMillis();
    System.out.printf("%s%n", filename);
    int balony = 0;
    try (NetcdfFile ncfile = open(filename)) {
      Sequence seq = (Sequence) ncfile.findVariable(RECORD);
      try (StructureDataIterator iter = seq.getStructureIterator(-1)) {
        while (iter.hasNext()) {
          StructureData sdata = iter.next();
          StructureMembers.Member m = sdata.findMember(YEAR);
          balony += sdata.getScalarInt(m);

        /* StructureMembers sm = sdata.getStructureMembers();
       for (StructureMembers.Member m : sm.getMembers()) {
         Array data = sdata.getArray(m);
         balony += data.getSize();
       } */
        }
      }
    }
    long took = System.currentTimeMillis() - start;
    System.out.printf("DONE %d == %d msecs%n", balony, took);
  }

  static public void main(String args[]) throws IOException {
    readData("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qcu.dat");
    readDataRegexp("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qcu.dat");
  }

      /////////////////////////////////////////////////////////////////////////////////
  // debug

  static private NetcdfFile open(String filename) throws IOException {
    Ghcnm iosp = new Ghcnm();
    RandomAccessFile raf = new RandomAccessFile(filename, "r");
    NetcdfFile ncfile = new NetcdfFileSubclass(iosp, filename);
    iosp.open(raf, ncfile, null);
    return ncfile;
  }

  static private void stnDuplicates(String filename, Set<Integer> stns, boolean wantDups) throws IOException {
    System.out.printf("%s%n", filename);
    int count = 0;
    int countDups = 0;
    NetcdfFile ncfile = open(filename);
    Sequence seq = (Sequence) ncfile.findVariable(STNS);
    try (StructureDataIterator iter = seq.getStructureIterator(-1)) {
      while (iter.hasNext()) {
        count++;
        StructureData sdata = iter.next();
        StructureMembers.Member m = sdata.findMember(STNID);
        int stnid = sdata.getScalarInt(m);
        if (stns.contains(stnid)) {
          countDups++;
          if (!wantDups) System.out.printf("  dup %d%n", stnid);
        } else {
          stns.add(stnid);
          if (wantDups) System.out.printf("  dup %d%n", stnid);
        }
      }
    }
    System.out.printf(" counts=%d dups=%d%n", count, countDups);
  }

  static public void main2(String args[]) throws IOException {
    Set<Integer> stns = new HashSet<Integer>(10 * 1000);
    stnDuplicates("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qae.inv", stns, false);
    stnDuplicates("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qca.inv", stns, true);
    stnDuplicates("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qcu.inv", stns, true);
  }

}

/*

  static public void main(String args[]) throws IOException {
    //InputStream is = cl.getResourceAsStream("resources/nj22/tables/nexrad.tbl");
    InputStream is = new FileInputStream("C:/data/ghcnm/ghcnm.v3.0.0-beta1.20101207.qae.inv");

    List<TableParser.Record> recs = TableParser.readTable(is, "11,20d,30d,37d,68,73i,74,79i,81,83,85,87i,88,90i,106,107", 10);
    Formatter f = new Formatter(System.out);
    //f.format("CNTRY WMO ID      YEAR  ELEM  VAL DM QC DS%n");
    for (TableParser.Record record : recs) {
      record.toString(f);
    }
  }
 */


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.unidata.geoloc.vertical;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import ucar.ma2.Array;
import ucar.ma2.ArrayDouble;
import ucar.ma2.ArrayDouble.D1;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;

/**
 * A subset of a vertical transform.
 *
 * @author  Unidata Development Team
 */
public class VerticalTransformSubset extends VerticalTransformImpl {

    private VerticalTransform original;
    private Range t_range;
    private List<Range> subsetList = new ArrayList<>();

    /**
     * Create a subset of an existing VerticalTransform
     * @param original make a subset of this
     * @param t_range subset the time dimension, or null if you want all of it
     * @param z_range subset the vertical dimension, or null if you want all of it
     * @param y_range subset the y dimension, or null if you want all of it
     * @param x_range subset the x dimension, or null if you want all of it
     */
    public VerticalTransformSubset(VerticalTransform original, Range t_range,
                                   Range z_range, Range y_range, Range x_range) {
        super(null);  // timeDim not used in this class

        this.original = original;
        this.t_range  = t_range;
        subsetList.add(z_range);
        subsetList.add(y_range);
        subsetList.add(x_range);

        units = original.getUnitString();
    }

    public ArrayDouble.D3 getCoordinateArray(int subsetIndex)
            throws IOException, InvalidRangeException {
        int orgIndex = subsetIndex;
        if (isTimeDependent() && (t_range != null)) {
            orgIndex = t_range.element(subsetIndex);
        }

        ArrayDouble.D3 data = original.getCoordinateArray(orgIndex);

        return (ArrayDouble.D3) data.sectionNoReduce(subsetList);
    }

    /**
     * Get the 1D vertical coordinate array for this time step and point
     *
     * @param timeIndex the time index. Ignored if !isTimeDependent().
     * @param xIndex    the x index
     * @param yIndex    the y index
     * @return vertical coordinate array
     * @throws java.io.IOException problem reading data
     * @throws ucar.ma2.InvalidRangeException _more_
     */
    public D1 getCoordinateArray1D(int timeIndex, int xIndex, int yIndex)
    		throws IOException, InvalidRangeException {

    	ArrayDouble.D3 data = original.getCoordinateArray(timeIndex);

    	int[] origin = new int[3];
    	int[] shape = new int[3];

    	shape[0] = subsetList.get(0).length();
    	shape[1] =1;
    	shape[2] =1;

    	origin[0] = timeIndex;
        if (isTimeDependent() && (t_range != null)) {
        	origin[0] = t_range.element(timeIndex);
        }

    	origin[1] = yIndex;
    	origin[2] = xIndex;

    	Array section = data.section(origin, shape);

    	return (ArrayDouble.D1) section.reduce();

    }

    public boolean isTimeDependent() {
        return original.isTimeDependent();
    }
}