/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.uf;

import ucar.nc2.constants.CDM;
import ucar.unidata.io.RandomAccessFile;
import ucar.ma2.Range;
import ucar.ma2.IndexIterator;

import java.nio.ByteBuffer;
import java.util.*;
import java.io.IOException;

public class Ray {
  int raySize;
  long rayOffset;
  static final int UF_MANDATORY_HEADER2_LEN = 90;
  static final int UF_FIELD_HEADER2_LEN = 50;
  boolean debug = false;

  /**
   * moment identifier
   */
  long data_msecs = 0;

  UF_mandatory_header2 uf_header2;
  UF_optional_header uf_opt_header;
  short numberOfFields;   // in this ray
  short numberOfRecords;  // in this ray
  short numberOfFieldsInRecord;   // in this record

  Map<String, UF_field_header2> field_header_map;  // key = 2 byte "data type"

  public Ray(ByteBuffer bos, int raySize, long rayOffset) {
    this.raySize = raySize;
    this.rayOffset = rayOffset;
    field_header_map = new HashMap<>();
    bos.position(0);

    byte[] data = new byte[UF_MANDATORY_HEADER2_LEN];
    bos.get(data);

    uf_header2 = new UF_mandatory_header2(data);

    if (uf_header2.offset2StartOfOptionalHeader > 0 &&
            (uf_header2.dataHeaderPosition != uf_header2.offset2StartOfOptionalHeader)) {
      data = new byte[28];
      bos.get(data);
      uf_opt_header = new UF_optional_header(data);
    }
    int position = uf_header2.dataHeaderPosition * 2 - 2;
    bos.position(position);
    data_msecs = setDateMesc();
    byte[] b2 = new byte[2];
    bos.get(b2);
    numberOfFields = getShort(b2, 0);
    bos.get(b2);
    numberOfRecords = getShort(b2, 0);
    bos.get(b2);
    numberOfFieldsInRecord = getShort(b2, 0);
    data = new byte[UF_FIELD_HEADER2_LEN];
    for (int i = 0; i < numberOfFields; i++) {
      bos.get(b2);
      //int type = getShort(b2, 0);
      String type = new String(b2, CDM.utf8Charset);
      bos.get(b2);
      int offs = getShort(b2, 0);
      int position0 = bos.position();
      bos.position(offs * 2 - 2);
      bos.get(data);
      UF_field_header2 field_header = new UF_field_header2(data);
      bos.position(position0);
      field_header_map.put(type, field_header);
    }
  }

  public int getRaySize() {
    return raySize;
  }

  public int getGateCount(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.binCount;
  }

  public String getDatatypeName(String abbrev) {
    if (abbrev.equals("ZN") || abbrev.equals("ZS"))
      return "Reflectivity";
    else if (abbrev.equals("ZF") || abbrev.equals("ZX") || abbrev.equals("DR"))
      return "Reflectivity";
    else if (abbrev.equals("VR") || abbrev.equals("DN") || abbrev.equals("DS") || abbrev.equals("DF") || abbrev.equals("DX"))
      return "RadialVelocity";
    else if (abbrev.equals("VN") || abbrev.equals("VF"))
      return "CorrectedRadialVelocity";
    else if (abbrev.equals("SW") || abbrev.equals("WS") || abbrev.equals("WF") || abbrev.equals("WX") || abbrev.equals("WN"))
      return "SpectrumWidth";
    else if (abbrev.equals("PN") || abbrev.equals("PS") || abbrev.equals("PF") || abbrev.equals("PX"))
      return "Power";
    else if (abbrev.equals("MN") || abbrev.equals("MS") || abbrev.equals("MF") || abbrev.equals("MX"))
      return "Power";
    else if (abbrev.equals("PH"))
      return "PhiDP";
    else if (abbrev.equals("RH"))
      return "RhoHV";
    else if (abbrev.equals("LH"))
      return "LdrH";
    else if (abbrev.equals("KD"))
      return "KDP";
    else if (abbrev.equals("LV"))
      return "LdrV";
    else if (abbrev.equals("DR"))
      return "ZDR";
    else if (abbrev.equals("CZ"))
      return "CorrecteddBZ";
    else if (abbrev.equals("DZ"))
      return "TotalReflectivity";
    else if (abbrev.equals("DR"))
      return "ZDR";
    else
      return abbrev;
  }

  public String getDatatypeUnits(String abbrev) {
    if (abbrev.equals("CZ") || abbrev.equals("DZ") || abbrev.equals("ZN") || abbrev.equals("ZS"))
      return "dBz";
    else if (abbrev.equals("ZF") || abbrev.equals("ZX"))
      return "dBz";
    else if (abbrev.equals("VR") || abbrev.equals("DN") || abbrev.equals("DS") || abbrev.equals("DF") || abbrev.equals("DX"))
      return "m/s";
    else if (abbrev.equals("VN") || abbrev.equals("VF"))
      return "m/s";
    else if (abbrev.equals("SW") || abbrev.equals("WS") || abbrev.equals("WF") || abbrev.equals("WX") || abbrev.equals("WN"))
      return "m/s";
    else if (abbrev.equals("PN") || abbrev.equals("PS") || abbrev.equals("PF") || abbrev.equals("PX"))
      return "dBM";
    else if (abbrev.equals("MN") || abbrev.equals("MS") || abbrev.equals("MF") || abbrev.equals("MX"))
      return "dBM";

    else
      return abbrev;
  }

  public short getDatatypeRangeFoldingThreshhold(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.thresholdValue;
  }

  public float getDatatypeScaleFactor(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return 1.0f / header.scaleFactor;
  }

  public float getDatatypeAddOffset(String abbrev) {
    return 0.0f;
  }

  public int getGateStart(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.startRange;
  }

  public int getDataOffset(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.dataOffset;

  }

  public int getGateSize(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.binSpacing;

  }

  public float getElevation() {
    return uf_header2.elevation / 64.f;
  }

  public float getAzimuth() {
    return uf_header2.azimuth / 64.f;
  }

  public short getMissingData() {
    return uf_header2.missing;
  }

  public int getYear() {
    return getYear(uf_header2.year);
  }

  public float getLatitude() {
    return uf_header2.latitudeD + (uf_header2.latitudeM + uf_header2.latitudeS / (64 * 60.f)) / 60.f;
  }

  public float getLongtitude() {
    return uf_header2.longitudeD + (uf_header2.longitudeM + uf_header2.longitudeS / (64 * 60.f)) / 60.f;
  }

  public float getHorizontalBeamWidth(String abbrev) {
    UF_field_header2 header = field_header_map.get(abbrev);
    return header.HorizontalBeamWidth / 64.f;

  }

  public int getYear(int year) {
    if (year > 1970)
      return year;
    if (year > 70 && year < 100)
      return 1900 + year;
    if (year < 60)
      return 2000 + year;

    return 0;
  }

  public long getTitleMsecs() {
    return data_msecs;

  }

  public long setDateMesc() {
    Calendar cal;
    if (uf_header2.timeZone.equals("UT") || uf_header2.timeZone.equals("GM")) {
      cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
    } else {
      cal = Calendar.getInstance();
    }
    cal.set(Calendar.YEAR, uf_header2.year);
    cal.set(Calendar.MONTH, uf_header2.month - 1);
    cal.set(Calendar.DAY_OF_MONTH, uf_header2.day);
    cal.set(Calendar.HOUR_OF_DAY, uf_header2.hour);
    cal.set(Calendar.MINUTE, uf_header2.minute);
    cal.set(Calendar.SECOND, uf_header2.second);

    return cal.getTimeInMillis();
  }

  public Date getDate() {
    Calendar cal = Calendar.getInstance();
    cal.setTimeInMillis(data_msecs);
    return cal.getTime();
  }

  class UF_mandatory_header2 {
    String textUF;
    short recordSize;   // in 16-bit words
    short offset2StartOfOptionalHeader; //, origin 1
    short localUseHeaderPosition;
    short dataHeaderPosition;
    short recordNumber;
    short volumeNumber;       // on tape, n/a for disk
    short rayNumber;          // within the volume scan
    short recordNumber1;      // within ray (origin 1)
    short sweepNumber;        // within the volume scan
    String radarName;          // char[8]
    String siteName;           // char[8]
    short latitudeD;          // degrees (North positive, South negative)
    short latitudeM;          // minutes
    short latitudeS;          // seconds*64
    short longitudeD;         // degrees (East positive, West negative)
    short longitudeM;         // Minutes
    short longitudeS;         // Seconds
    short height;             // of antenna above sea level in meters
    short year;               // (time of data acquisition)
    short month;
    short day;
    short hour;
    short minute;
    short second;
    String timeZone;            // UT for universal  char[2]
    short azimuth;            // (degrees*64) of midpoint of sample
    short elevation;          // (degrees*64)
    short sweepMode;
    //   0:Cal       1:PPI       2:Coplane 3:RHI
    //   4:Vertical 5:Target 6:Manual 7:Idle
    short fixedAngle;         // (degrees*64)
    short sweepRate;          // ((degrees/second)*64)
    short year1;              // (generation data of UF format)
    short month1;

    short day1;
    String nameOfUFGeneratorProgram; //  char[8]
    short missing;             // Value stored for deleted or missing data (0x8000)

    UF_mandatory_header2(byte[] data) {
      // data is of length 90 bytes
      textUF = new String(data, 0, 2, CDM.utf8Charset);
      if (debug) {
        System.out.println(textUF);
      }
      recordSize = getShort(data, 2);
      offset2StartOfOptionalHeader = getShort(data, 4);
      localUseHeaderPosition = getShort(data, 6);
      dataHeaderPosition = getShort(data, 8);
      recordNumber = getShort(data, 10);
      volumeNumber = getShort(data, 12);
      rayNumber = getShort(data, 14);
      recordNumber1 = getShort(data, 16);
      sweepNumber = getShort(data, 18);
      radarName = new String(data, 20, 8, CDM.utf8Charset).trim();
      siteName = new String(data, 28, 8, CDM.utf8Charset).trim();
      latitudeD = getShort(data, 36);
      latitudeM = getShort(data, 38);
      latitudeS = getShort(data, 40);
      longitudeD = getShort(data, 42);
      longitudeM = getShort(data, 44);
      longitudeS = getShort(data, 46);
      height = getShort(data, 48);
      int yearValue = getShort(data, 50);
      year = (short) getYear(yearValue);
      month = getShort(data, 52);
      day = getShort(data, 54);
      hour = getShort(data, 56);
      minute = getShort(data, 58);
      second = getShort(data, 60);
      timeZone = new String(data, 62, 2, CDM.utf8Charset);
      azimuth = getShort(data, 64);
      elevation = getShort(data, 66);
      sweepMode = getShort(data, 68);
      fixedAngle = getShort(data, 70);         // (degrees*64)
      sweepRate = getShort(data, 72);          // ((degrees/second)*64)
      year1 = getShort(data, 74);              // (generation data of UF format)
      month1 = getShort(data, 76);
      day1 = getShort(data, 78);
      nameOfUFGeneratorProgram = new String(data, 80, 8,
              CDM.utf8Charset); //  char[8]
      missing = getShort(data, 88);             // Value stored for deleted or missing data (0x8000)

    }

  }

  class UF_optional_header {

    String sProjectName;   //  char[8]
    short iBaselineAzimuth;
    short iBaselineelevation;
    short iVolumeScanHour;  /* Time of start of current volume scan */
    short iVolumeScanMinute;
    short iVolumeScanSecond;
    String sFieldTapeName;  // char[8]
    short iFlag;

    UF_optional_header(byte[] data) {
      sProjectName = new String(data, 0, 8, CDM.utf8Charset);
      iBaselineAzimuth = getShort(data, 8);
      iBaselineelevation = getShort(data, 10);
      iVolumeScanHour = getShort(data, 12);
      iVolumeScanMinute = getShort(data, 14);
      iVolumeScanSecond = getShort(data, 16);
      sFieldTapeName = new String(data, 18, 8, CDM.utf8Charset);
      iFlag = getShort(data, 26);
    }

  }

  class UF_field_header2 {

    short dataOffset;   // from start of record, origin 1
    short scaleFactor;  // met units = file value/scale
    short startRange;   // km
    short startRange1;   // meters
    short binSpacing;  // in meters
    short binCount;
    short pulseWidth;  // in meters
    short HorizontalBeamWidth; // in degrees*64
    short verticalBeamWidth; // in degrees*64
    short receiverBandwidth; // in Mhz*64 ?
    short polarization;  //:          1:horz   2:vert
    short waveLength;  // in cm*64
    short sampleSize;
    String typeOfData;     //used to threshold  //  char[2]
    short thresholdValue;
    short scale;
    String editCode; //  char[2]
    short prt;  // in microseconds
    short bits; // per bin, must be 16
    //         38         12      <uf_fsi2>

    UF_field_header2(byte[] data) {
      dataOffset = getShort(data, 0);
      scaleFactor = getShort(data, 2);
      startRange = getShort(data, 4);
      startRange1 = getShort(data, 6);
      binSpacing = getShort(data, 8);
      binCount = getShort(data, 10);
      pulseWidth = getShort(data, 12);
      HorizontalBeamWidth = getShort(data, 14);
      verticalBeamWidth = getShort(data, 16);
      receiverBandwidth = getShort(data, 18);
      polarization = getShort(data, 20);
      waveLength = getShort(data, 22);
      sampleSize = getShort(data, 24);
      typeOfData = new String(data, 26, 2, CDM.utf8Charset);
      thresholdValue = getShort(data, 28);
      scale = getShort(data, 30);
      editCode = new String(data, 32, 2, CDM.utf8Charset);
      prt = getShort(data, 34);
      bits = getShort(data, 36);
    }
  }

  protected short getShort(byte[] bytes, int offset) {

    // careful that we only allow sign extension on the highest order byte
    return (short) bytesToShort(bytes[offset], bytes[offset + 1], false);
  }

  public static int bytesToShort(byte a, byte b, boolean swapBytes) {
    // again, high order bit is expressed left into 32-bit form
    if (swapBytes) {
      return (a & 0xff) + ((int) b << 8);
    } else {
      return ((int) a << 8) + (b & 0xff);
    }
  }

  public short[] byte2short(byte[] a, int length) {
    int len = length / 2;
    short[] b = new short[len];
    byte[] b2 = new byte[2];

    for (int i = 0; i < len; i++) {
      b2[0] = a[2 * i];
      b2[1] = a[2 * i + 1];
      b[i] = getShort(b2, 0);
    }

    return b;
  }

  /**
   * Read data from this ray.
   *
   * @param raf       read from this file
   * @param abbrev    which data type we want
   * @param gateRange handles the possible subset of data to return
   * @param ii        put the data here
   * @throws java.io.IOException
   */
  public void readData(RandomAccessFile raf, String abbrev, Range gateRange, IndexIterator ii) throws IOException {
    long offset = rayOffset;
    offset += (getDataOffset(abbrev) * 2 - 2);
    raf.seek(offset);
    byte[] b2 = new byte[2];
    int dataCount = getGateCount(abbrev);
    byte[] data = new byte[dataCount * 2];
    raf.readFully(data);

    for (int gateIdx : gateRange) {
      if (gateIdx >= dataCount)
        ii.setShortNext(uf_header2.missing);
      else {
        b2[0] = data[gateIdx * 2];
        b2[1] = data[gateIdx * 2 + 1];
        short value = getShort(b2, 0);

        ii.setShortNext(value);
      }
    }

  }

}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import java.io.*;
import java.util.Enumeration;
import java.util.Vector;

/**
 * A DStructure in OPeNDAP can hold N instances of any of
 * the other datatypes, including other structures.
 *
 * @author jehamby
 * @version $Revision: 15901 $
 * @see BaseType
 * @see DConstructor
 */
public class DStructure extends DConstructor implements ClientIO {
    /**
     * The variables in this DStructure, stored in a
     * Vector of BaseType objects.
     */
    protected Vector vars;

    /**
     * Constructs a new DStructure.
     */
    public DStructure() {
        this("");
    }

    /**
     * Constructs a new DStructure with name n.
     *
     * @param n the name of the variable.
     */
    public DStructure(String n) {
        super(n);
        vars = new Vector();
    }

    /**
     * Returns the OPeNDAP type name of the class instance as a String.
     *
     * @return the OPeNDAP type name of the class instance as a String.
     */
    public String getTypeName() {
        return "Structure";
    }

    /**
     * Returns the number of variables contained in this object. For simple and
     * vector type variables, it always returns 1. To count the number
     * of simple-type variable in the variable tree rooted at this variable, set
     * leaves to true.
     *
     * @param leaves If true, count all the simple types in the `tree' of
     *               variables rooted at this variable.
     * @return the number of contained variables.
     */
    public int elementCount(boolean leaves) {
        if (!leaves)
            return vars.size();
        else {
            int count = 0;
            for (Enumeration e = vars.elements(); e.hasMoreElements();) {
                BaseType bt = (BaseType) e.nextElement();
                count += bt.elementCount(leaves);
            }
            return count;
        }
    }

    /**
     * Adds a variable to the container.
     *
     * @param v    the variable to add.
     * @param part ignored for DSequence.
     */
    public void addVariable(BaseType v, int part) {
        v.setParent(this);
        vars.addElement(v);
    }

    /**
     * Returns the named variable.
     *
     * @param name the name of the variable.
     * @return the named variable.
     * @throws NoSuchVariableException if the named variable does not
     *                                 exist in this container.
     */
    public BaseType getVariable(String name) throws NoSuchVariableException {
        int dotIndex = name.indexOf('.');
        if (dotIndex != -1) {  // name contains "."
            String aggregate = name.substring(0, dotIndex);
            String field = name.substring(dotIndex + 1);

            BaseType aggRef = getVariable(aggregate);
            if (aggRef instanceof DConstructor)
                return ((DConstructor) aggRef).getVariable(field);  // recurse
            else
                ; // fall through to throw statement
        } else {
            for (Enumeration e = vars.elements(); e.hasMoreElements();) {
                BaseType v = (BaseType) e.nextElement();
                if (v.getEncodedName().equals(name))
                    return v;
            }
        }
        throw new NoSuchVariableException("DStructure: getVariable()");
    }

    /**
     * Gets the indexed variable. For a DStructure this returns the
     * BaseType from the indexth column from the
     * internal storage Vector.
     *
     * @param index the index of the variable in the Vector Vars.
     * @return the indexed variable.
     * @throws NoSuchVariableException if the named variable does not
     *                                 exist in this container.
     */
    public BaseType getVar(int index)
            throws NoSuchVariableException {

        if (index < vars.size())
            return ((BaseType) vars.elementAt(index));
        else
            throw new NoSuchVariableException(
                    "DStructure.getVariable(" + index + " - 1)");
    }

    /**
     * Get the number of contained variables (for use with getVar()
     * @return the number of contained variables
     */
    public int getVarCount()
    {
	return vars.size();
    }

    /**
     * Return an Enumeration that can be used to iterate over the members of
     * a Structure. This implementation provides access to the elements of
     * the Structure. Each Object returned by the Enumeration can be cast to
     * a BaseType.
     *
     * @return An Enumeration
     */
    public Enumeration getVariables() {
        return vars.elements();
    }

    /**
     * Return an the Vector of vars for the structure
     * @return A Vector of BaseType
     */
    public Vector getVariableSet() {
        return vars;
    }

    /**
     * Checks for internal consistency. For DStructure, verify
     * that the variables have unique names.
     *
     * @param all for complex constructor types, this flag indicates whether to
     *            check the semantics of the member variables, too.
     * @throws BadSemanticsException if semantics are bad, explains why.
     * @see BaseType#checkSemantics(boolean)
     */
    public void checkSemantics(boolean all)
            throws BadSemanticsException {
        super.checkSemantics(all);

        Util.uniqueNames(vars, getEncodedName(), getTypeName());

        if (all) {
            for (Enumeration e = vars.elements(); e.hasMoreElements();) {
                BaseType bt = (BaseType) e.nextElement();
                bt.checkSemantics(true);
            }
        }
    }

    /**
     * Write the variable's declaration in a C-style syntax. This
     * function is used to create textual representation of the Data
     * Descriptor Structure (DDS).  See The OPeNDAP User Manual for
     * information about this structure.
     *
     * @param os         The PrintWriter on which to print the
     *                   declaration.
     * @param space      Each line of the declaration will begin with the
     *                   characters in this string.  Usually used for leading spaces.
     * @param print_semi a boolean value indicating whether to print a
     *                   semicolon at the end of the declaration.
     * @see BaseType#printDecl(PrintWriter, String, boolean)
     */
    //Coverity[CALL_SUPER]
    public void printDecl(PrintWriter os, String space,
                          boolean print_semi, boolean constrained) {

        // BEWARE! Since printDecl()is (multiple) overloaded in BaseType and
        // all of the different signatures of printDecl() in BaseType lead to
        // one signature, we must be careful to override that SAME signature
        // here. That way all calls to printDecl() for this object lead to
        // this implementation.

        os.println(space + getTypeName() + " {");
        for (Enumeration e = vars.elements(); e.hasMoreElements();) {
            BaseType bt = (BaseType) e.nextElement();
            bt.printDecl(os, space + "    ", true, constrained);
        }
        os.print(space + "} " + getEncodedName());

        if (print_semi)
            os.println(";");

    }

    /**
     * Prints the value of the variable, with its declaration.  This
     * function is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *
     * @param os           the PrintWriter on which to print the value.
     * @param space        this value is passed to the printDecl method,
     *                     and controls the leading spaces of the output.
     * @param print_decl_p a boolean value controlling whether the
     *                     variable declaration is printed as well as the value.
     * @see BaseType#printVal(PrintWriter, String, boolean)
     */
    public void printVal(PrintWriter os, String space, boolean print_decl_p) {
        if (print_decl_p) {
            printDecl(os, space, false);
            os.print(" = ");
        }

        os.print("{ ");
        for (Enumeration e = vars.elements(); e.hasMoreElements();) {
            BaseType bt = (BaseType) e.nextElement();
            bt.printVal(os, "", false);
            if (e.hasMoreElements())
                os.print(", ");
        }
        os.print(" }");

        if (print_decl_p)
            os.println(";");
    }

    /**
     * Reads data from a DataInputStream. This method is only used
     * on the client side of the OPeNDAP client/server connection.
     *
     * @param source   a DataInputStream to read from.
     * @param sv       the ServerVersion returned by the server.
     * @param statusUI the StatusUI object to use for GUI updates
     *                 and user cancellation notification (may be null).
     * @throws EOFException      if EOF is found before the variable is completely
     *                           deserialized.
     * @throws IOException       thrown on any other InputStream exception.
     * @throws DataReadException if an unexpected value was read.
     * @see ClientIO#deserialize(DataInputStream, ServerVersion, StatusUI)
     */
    public synchronized void deserialize(DataInputStream source,
                                         ServerVersion sv,
                                         StatusUI statusUI)
            throws IOException, EOFException, DataReadException {
        for (Enumeration e = vars.elements(); e.hasMoreElements();) {
            if (statusUI != null && statusUI.userCancelled())
                throw new DataReadException("User cancelled");
            ClientIO bt = (ClientIO) e.nextElement();
            bt.deserialize(source, sv, statusUI);
        }
    }

    /**
     * Writes data to a DataOutputStream. This method is used
     * primarily by GUI clients which need to download OPeNDAP data, manipulate
     * it, and then re-save it as a binary file.
     *
     * @param sink a DataOutputStream to write to.
     * @throws IOException thrown on any OutputStream
     *                     exception.
     */
    public void externalize(DataOutputStream sink) throws IOException {
        for (Enumeration e = vars.elements(); e.hasMoreElements();) {
            ClientIO bt = (ClientIO) e.nextElement();
            bt.externalize(sink);
        }
    }

    /**
     * Returns a clone of this Structure.
     * See DAPNode.cloneDag()
     *
     * @param map track previously cloned nodes
     * @return a clone of this object.
     */
    public DAPNode cloneDAG(CloneMap map)
        throws CloneNotSupportedException
    {
        DStructure s = (DStructure) super.cloneDAG(map);
        s.vars = new Vector();
        for (int i = 0; i < vars.size(); i++) {
            BaseType bt = (BaseType) vars.elementAt(i);
	        BaseType btclone = (BaseType)cloneDAG(map,bt);
            s.vars.addElement(btclone);
        }
        return s;
    }

}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import java.io.*;

/**
 * A vector of doubles.
 *
 * @author ndp
 * @version $Revision: 20918 $
 * @see PrimitiveVector
 */
public class Float32PrimitiveVector extends PrimitiveVector
{

    static final long serialVersionUID = 1;

    /**
     * the array of float values.
     */
    private float vals[];

    /**
     * Constructs a new Float32PrimitiveVector.
     *
     * @param var the template BaseType to use.
     */
    public Float32PrimitiveVector(BaseType var) {
        super(var);
    }

    /**
     * Returns the number of elements in the array.
     *
     * @return the number of elements in the array.
     */
    public int getLength() {
        return vals.length;
    }

    /**
     * Sets the number of elements in the array.  Allocates a new primitive
     * array of the desired size.  Note that if this is called multiple times,
     * the old array and its contents will be lost.
     *

     * Only called inside of deserialize method or in derived
     * classes on server.
     *
     * @param len the number of elements in the array.
     */
    public void setLength(int len) {
        vals = new float[len];
    }

    /**
     * Return the i'th value as a double.
     *
     * @param i the index of the value to return.
     * @return the i'th value.
     */
    public final float getValue(int i) {
        return vals[i];
    }

    /**
     * Set the i'th value of the array.
     *
     * @param i      the index of the value to set.
     * @param newVal the new value.
     */
    public final void setValue(int i, float newVal) {
        vals[i] = newVal;
    }

    /**
     * Prints the value of all variables in this vector.  This
     * method is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *
     * @param os    the PrintWriter on which to print the value.
     * @param space this value is passed to the printDecl method,
     *              and controls the leading spaces of the output.
     * @see BaseType#printVal(PrintWriter, String, boolean)
     */
    public void printVal(PrintWriter os, String space) {
        int len = vals.length;
        for (int i = 0; i < len - 1; i++) {
            os.print(vals[i]);
            os.print(", ");
        }
        // print last value, if any, without trailing comma
        if (len > 0)
            os.print(vals[len - 1]);
    }

    /**
     * Prints the value of a single variable in this vector.
     * method is used by DArray's printVal method.
     *
     * @param os    the PrintWriter on which to print the value.
     * @param index the index of the variable to print.
     * @see DArray#printVal(PrintWriter, String, boolean)
     */
    public void printSingleVal(PrintWriter os, int index) {
        os.print(vals[index]);
    }

    /**
     * Reads data from a DataInputStream. This method is only used
     * on the client side of the OPeNDAP client/server connection.
     *
     * @param source   a DataInputStream to read from.
     * @param sv       The ServerVersion returned by the server.
     *                 (used by DSequence to determine which protocol version was
     *                 used).
     * @param statusUI The StatusUI object to use for GUI updates
     *                 and user cancellation notification (may be null).
     * @throws DataReadException when invalid data is read, or if the user
     *                           cancels the download.
     * @throws EOFException      if EOF is found before the variable is completely
     *                           deserialized.
     * @throws IOException       thrown on any other InputStream exception.
     * @see ClientIO#deserialize(DataInputStream, ServerVersion, StatusUI)
     */
    public synchronized void deserialize(DataInputStream source,
                                         ServerVersion sv,
                                         StatusUI statusUI)
            throws IOException, EOFException, DataReadException {
        for (int i = 0; i < vals.length; i++) {
            vals[i] = source.readFloat();

           //LogStream.out.printf("%d:%f ",i,vals[i]);
            if (statusUI != null) {
                statusUI.incrementByteCount(8);
                if (statusUI.userCancelled())
                    throw new DataReadException("User cancelled");
            }
        }
    }

    /**
     * Writes data to a DataOutputStream. This method is used
     * primarily by GUI clients which need to download OPeNDAP data, manipulate
     * it, and then re-save it as a binary file.
     *
     * @param sink a DataOutputStream to write to.
     * @throws IOException thrown on any OutputStream
     *                     exception.
     */
    public void externalize(DataOutputStream sink) throws IOException {
        for (int i = 0; i < vals.length; i++) {
            sink.writeFloat(vals[i]);
        }
    }

    /**
     * Write a subset of the data to a DataOutputStream.
     *
     * @param sink    a DataOutputStream to write to.
     * @param start  starting index (i=start)
     * @param stop   ending index (i<=stop)
     * @param stride index stride (i+=stride)
     * @throws IOException thrown on any OutputStream exception.
     */
    public void externalize(DataOutputStream sink, int start, int stop, int stride) throws IOException {
        for (int i = start; i <= stop; i += stride)
            sink.writeFloat(vals[i]);
    }

    /**
     * Returns (a reference to) the internal storage for this PrimitiveVector
     * object.
     *
WARNING:

     * Because this method breaks encapsulation rules the user must beware!
     * If we (the OPeNDAP prgramming team) choose to change the internal
     * representation(s) of these types your code will probably break.
     *

     * This method is provided as an optimization to eliminate massive
     * copying of data.
     *
     * @return The internal array of floats.
     */
    public Object getInternalStorage() {
        return (vals);
    }

    /**
     * Set the internal storage for PrimitiveVector.
     *
WARNING:

     * Because this method breaks encapsulation rules the user must beware!
     * If we (the OPeNDAP prgramming team) choose to change the internal
     * representation(s) of these types your code will probably break.
     *

     * This method is provided as an optimization to eliminate massive
     * copying of data.
     */
    public void setInternalStorage(Object o) {
        vals = (float []) o;
    }

    /**
     * Create a new primitive vector using a subset of the data.
     *
     * @param start  starting index (i=start)
     * @param stop   ending index (i<=stop)
     * @param stride index stride (i+=stride)
     * @return new primitive vector, of type Float32PrimitiveVector.
     */
    public PrimitiveVector subset(int start, int stop, int stride) {
        Float32PrimitiveVector n = new Float32PrimitiveVector(getTemplate());
        stride = Math.max(stride, 1);
        stop = Math.max(start, stop);
        int length = 1 + (stop - start) / stride;
        n.setLength(length);

        int count = 0;
        for (int i = start; i <= stop; i += stride) {
            n.setValue(count, vals[i]);
            count++;
        }
        return n;
    }

/**
   * Prints the value of the variable, with its declaration.  This
   * function is primarily intended for debugging OPeNDAP applications and
   * text-based clients such as geturl.
   *
   * @param os           the PrintWriter on which to print the value.
   * @param space        this value is passed to the printDecl method,
   *                     and controls the leading spaces of the output.
   * @param print_decl_p a boolean value controlling whether the
   *                     variable declaration is printed as well as the value.
   */
   public void printVal(PrintWriter os, String space,
                                boolean print_decl_p) {}

    /**
     * Returns a clone of this PrimitiveVector.
     * See DAPNode.cloneDag()
     *
     * @param map track previously cloned nodes
     * @return a clone of this Float32PrimitiveVector.
     */
    public DAPNode cloneDAG(CloneMap map)
        throws CloneNotSupportedException
    {
        Float32PrimitiveVector v = (Float32PrimitiveVector) super.cloneDAG(map);
        if (vals != null) {
            v.vals = new float[vals.length];
            System.arraycopy(vals, 0, v.vals, 0, vals.length);
        }
        return v;
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib2;

import javax.annotation.Nullable;
import ucar.nc2.constants.AxisType;
import ucar.nc2.grib.GribTables;
import ucar.nc2.grib.grib2.table.Grib2Tables;
import ucar.nc2.grib.grib2.table.WmoParamTable;
import ucar.nc2.time.CalendarPeriod;
import ucar.nc2.units.SimpleUnit;
import ucar.nc2.wmo.Util;
import ucar.unidata.util.StringUtil2;

import java.util.Formatter;
import java.util.List;

/**
 * Static utilities for Grib-2
 *
 * @author caron
 * @since 3/29/11
 */
public class Grib2Utils {

  public static String clean(String s) {
    StringBuilder sb = new StringBuilder(s);
    StringUtil2.replace(sb, "/. ", "-p_");
    StringUtil2.remove(sb, "(),;");
    char c = sb.charAt(0);
    if (Character.isLetter(c)) {
      if (Character.isLowerCase(c))
        sb.setCharAt(0, Character.toUpperCase(c));
    } else {
      sb.insert(0, 'N');
    }

    return sb.toString().trim();
  }

  public static String cleanupHeader(byte[] raw) {
    String result = StringUtil2.cleanup(raw);
    int pos = result.indexOf("data");
    if (pos > 0) result = result.substring(pos);
    return result;
  }

  public static String getVariableName(Grib2Record gr) {
    GribTables.Parameter p = WmoParamTable.getParameter(gr.getDiscipline(), gr.getPDS().getParameterCategory(), gr.getPDS().getParameterNumber());
    String s = (p == null) ? null : p.getName();
    if (s == null)
      s = "U"+ gr.getDiscipline()+"-"+gr.getPDS().getParameterCategory()+"-" + gr.getPDS().getParameterNumber();
    return s;
  }

  @Nullable
  public static CalendarPeriod getCalendarPeriod(int timeUnit) {

    switch (timeUnit) { // code table 4.4
      case 0:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Minute);
      case 1:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Hour);
      case 2:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Day);
      case 3:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Month);
      case 4:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Year);
      case 5:
        return CalendarPeriod.of(10, CalendarPeriod.Field.Year);
      case 6:
        return CalendarPeriod.of(30, CalendarPeriod.Field.Year);
      case 7:
        return CalendarPeriod.of(100, CalendarPeriod.Field.Year);
      case 10:
        return CalendarPeriod.of(3, CalendarPeriod.Field.Hour);
      case 11:
        return CalendarPeriod.of(6, CalendarPeriod.Field.Hour);
      case 12:
        return CalendarPeriod.of(12, CalendarPeriod.Field.Hour);
      case 13:
        return CalendarPeriod.of(1, CalendarPeriod.Field.Second);
      default:
        return null;
    }
  }

  /**
   * Check to see if this pds is a layer variable
   * @param pds record to check
   * @return true if a layer
   */
  public static boolean isLayer(Grib2Pds pds) {
    return pds.getLevelType2() != 255 && pds.getLevelType2() != 0;
  }

  public static boolean isLatLon(int gridTemplate, int center) {
    return ((gridTemplate < 4) || ((gridTemplate >= 40) && (gridTemplate < 44)));
  }

  //////////////////////////////////////////////////////////////////////////////////
  // pretty much lame stuff
  // possibly move to Customizer

  // check if grid template is "Curvilinear Orthogonal", (NCEP 204) methods below only used when thats true
  public static boolean isCurvilinearOrthogonal(int gridTemplate, int center) {
    return ((center == 7) && (gridTemplate == 204));
  }

  // isLatLon2D is true, check parameter to see if its a 2D lat/lon coordinate
  @Nullable
  public static LatLon2DCoord getLatLon2DcoordType(int discipline, int category, int parameter) {
    if ((discipline != 0) || (category != 2) || (parameter < 198 || parameter > 203)) return null;
    switch (parameter) {
      case 198:
        return LatLon2DCoord.U_Latitude;
      case 199:
         return LatLon2DCoord.U_Longitude;
      case 200:
        return LatLon2DCoord.V_Latitude;
      case 201:
        return LatLon2DCoord.V_Longitude;
      case 202:
       return LatLon2DCoord.P_Latitude;
      case 203:
       return LatLon2DCoord.P_Longitude;
    }
    return null;
  }

  public enum LatLonCoordType {U, V, P }
  public enum LatLon2DCoord {
    U_Latitude, U_Longitude, V_Latitude, V_Longitude, P_Latitude, P_Longitude;

    public AxisType getAxisType() {
      return this.name().contains("Latitude") ? AxisType.Lat : AxisType.Lon;
    }
  }

  /**
   * This looks for snippets in the variable name/desc as to whether it wants U, V, or P 2D coordinates
   * @param desc variable name/desc
   * @return  U, V, or P for normal variables, null for the coordinates themselves
   */
  public static LatLonCoordType getLatLon2DcoordType(String desc) {
    LatLonCoordType type;
    if (desc.contains("u-component")) type = LatLonCoordType.U;
    else if (desc.contains("v-component")) type = LatLonCoordType.V;
    else if (desc.contains("Latitude of") || desc.contains("Longitude of")) type = null;
    else type = LatLonCoordType.P;
    return type;
  }

  // Compare 2 tables, print report.
  public static void compareTables(String name1, String name2, List<? extends GribTables.Parameter> test, Grib2Tables reference, Formatter f) {

    int extra = 0;
    int udunits = 0;
    int conflict = 0;
    f.format("Table 1 : %s%n", name1);
    f.format("Table 2 : %s%n", name2);
    for (GribTables.Parameter p1 : test) {
      GribTables.Parameter  p2 = reference.getParameter(p1.getDiscipline(), p1.getCategory(), p1.getNumber());
      if (p2 == null) {
        if (p1.getCategory() < 192 && p1.getNumber() < 192) {
          extra++;
          f.format("  WMO missing %s%n", p1);
        }

      } else {
        String p1n = Util.cleanName(p1.getName());
        String p2n = Util.cleanName(p2.getName());

        if (!p1n.equalsIgnoreCase(p2n)) {
          f.format("  p1=%10s %40s %15s %15s %s%n", p1.getId(), p1.getName(), p1.getUnit(), p1.getAbbrev(), p1.getDescription());
          f.format("  p2=%10s %40s %15s %15s %s%n%n", p2.getId(), p2.getName(), p2.getUnit(), p2.getAbbrev(), p2.getDescription());
          conflict++;
        }

        if (!p1.getUnit().equalsIgnoreCase(p2.getUnit())) {
          String cu1 = Util.cleanUnit(p1.getUnit());
          String cu2 = Util.cleanUnit(p2.getUnit());

          // eliminate common non-udunits
          boolean isUnitless1 = Util.isUnitless(cu1);
          boolean isUnitless2 = Util.isUnitless(cu2);

          if (isUnitless1 != isUnitless2) {
            f.format("  ud=%10s %s != %s for %s (%s)%n%n", p1.getId(), cu1, cu2, p1.getId(), p1.getName());
            udunits++;

          } else if (!isUnitless1) {

            try {
              SimpleUnit su1 = SimpleUnit.factoryWithExceptions(cu1);
              if (!su1.isCompatible(cu2)) {
                f.format("  ud=%10s %s (%s) != %s for %s (%s)%n%n", p1.getId(), cu1, su1, cu2, p1.getId(), p1.getName());
                udunits++;
              }
            } catch (Exception e) {
              f.format("  udunits cant parse=%10s %15s %15s%n", p1.getId(), cu1, cu2);
            }
          }

        }
      }
    }
    f.format("Conflicts=%d extra=%d udunits=%d%n%n", conflict, extra, udunits);

    f.format("Parameters in %s not in %s%n", name1, name2);
    int local = 0;
    for (GribTables.Parameter p1 : test) {
      GribTables.Parameter  p2 = reference.getParameter(p1.getDiscipline(), p1.getCategory(), p1.getNumber());
      if (p2 == null) {
        local++;
        f.format("  %s%n", p1);
      }
    }
    f.format(" missing=%d%n%n", local);
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.ma2;

/**
 * Abstraction for vector operations.
 * A vector is a rank-1 Array.
 * All operations done in double precision
 *
 * @author @caron
 */

public class MAVector {
  private Array a;
  private int nelems;
  private Index ima;

  /**
   * Create an MAVector out of a double array
   */
  public MAVector( double[] values) {
    this( Array.makeFromJavaArray(values, false));
  }

  /**
   * Create an MAVector of the given length.
   */
  public MAVector( int nelems) {
    this.a = new ArrayDouble.D1(nelems);
    this.nelems = nelems;
    ima = a.getIndex();
  }

  /**
   * Create an MAVector using the given rank-1 array.
   * @param a rank-1 array
   * @exception IllegalArgumentException is a is not rank 1
   */
  public MAVector( Array a) {
    this.a = a;
    if (a.getRank() != 1)
      throw new IllegalArgumentException("rank != 1, instead = "+ a.getRank());
    nelems = a.getShape()[0];
    ima = a.getIndex();
  }

  public int getNelems() { return nelems; }
  public double getDouble(int i) { return a.getDouble(ima.set(i)); }
  public void setDouble(int i, double val) { a.setDouble(ima.set(i), val); }

  /**
   * Create a new MAVector that is the same as this one, with a copy of the backing store.
   */
  public MAVector copy() {
    return new MAVector( a.copy());
  }

  /**
   * Cos between two vectors = dot(v) / norm() * norm(v)
   * @param v cosine with this vector
   * @return double result: cos between this and v
   * @exception IllegalArgumentException if nelems != v.getNelems().
   */
  public double cos(MAVector v) {

    if (nelems != v.getNelems())
      throw new IllegalArgumentException("MAVector.cos "+nelems+" != "+ v.getNelems());

    double norm = norm();
    double normV = v.norm();
    if ((norm == 0.0) || (normV == 0.0))
      return 0.0;
    else
      return dot(v)/(norm*normV);
  }

  /**
   * Dot product of 2 vectors
   * @param v dot product with this vector
   * @return double result: dot product
   * @exception IllegalArgumentException if nelems != v.getNelems().
   */
  public double dot(MAVector v) {

    if (nelems != v.getNelems())
      throw new IllegalArgumentException("MAVector.dot "+nelems+" != "+ v.getNelems());

    double sum = 0.0;
    for (int k=0; k<nelems; k++)
      sum += getDouble(k) * v.getDouble(k);

    return sum;
  }

  /**
   * Get the L2 norm of this vector.
   * @return double norm
   */
  public double norm() {

    double sum = 0.0;
    for (int k=0; k<nelems; k++) {
      double val = getDouble(k);
      sum += val * val;
    }

    return Math.sqrt(sum);
  }

  /**
   * Normalize this vector, so it has norm = 1.0.
   */
  public void normalize() {
    double norm = norm();
    if (norm <= 0.0)
      return;

    for (int k=0; k<nelems; k++) {
      double val = getDouble(k);
      setDouble(k, val/norm);
    }
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.client.catalog.builder;

import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.input.SAXBuilder;
import thredds.client.catalog.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.DataFormatType;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.time.Calendar;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateFormatter;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;
import ucar.nc2.util.URLnaming;
import ucar.unidata.util.StringUtil2;

import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

/**
 * Builds client Catalogs using JDOM2
 * Non validating.
 *
 * @author caron
 * @since 1/8/2015
 */
public class CatalogBuilder {
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CatalogBuilder.class);

  //////////////////////////////////////////////////////////////////////////////////
  // used when reading from XML
  private Map<String, Service> serviceMap = new HashMap<>(); // LOOK why not instead of services ?
  protected Formatter errlog = new Formatter();
  protected boolean fatalError = false;

  public Catalog buildFromLocation(String location, URI baseURI) throws IOException {
    location = StringUtil2.replace(location, "\\", "/");

    if (baseURI == null) {
      try {
        baseURI = new URI(location);
      }
      catch (URISyntaxException e) {
        errlog.format("Bad location = '%s' err='%s'%n", location, e.getMessage());
        logger.error("Bad location = '{}' err='{}}'", location, e.getMessage());
        fatalError = true;
        return null;
      }
    }

    this.baseURI = baseURI;
    readXML(location);
    return fatalError ? null : makeCatalog();
  }

  public Catalog buildFromURI(URI uri) throws IOException {
    this.baseURI = uri;
    readXML(uri);
    return fatalError ? null : makeCatalog();
  }

  public Catalog buildFromCatref(CatalogRef catref) throws IOException {
    URI catrefURI = catref.getURI();
    if (catrefURI == null) {
      errlog.format("Catref doesnt have valid UrlPath=%s%n", catref.getUrlPath());
      logger.error("Catref doesnt have valid UrlPath={}", catref.getUrlPath());
      fatalError = true;
      return null;
    }
    this.baseURI = catrefURI;
    Catalog result =  buildFromURI(catrefURI);
    catref.setRead(!fatalError);
    return fatalError ? null : result;
  }

  public Catalog buildFromString(String catalogAsString, URI docBaseUri) throws IOException {
    this.baseURI = docBaseUri;
    readXMLfromString(catalogAsString);
    return fatalError ? null : makeCatalog();
  }

  public Catalog buildFromStream(InputStream stream, URI docBaseUri) throws IOException {
    this.baseURI = docBaseUri;
    readXML(stream);
    return fatalError ? null : makeCatalog();
  }

  public Catalog buildFromJdom(Element root, URI docBaseUri) throws IOException {
    this.baseURI = docBaseUri;
    readCatalog(root);
    return fatalError ? null : makeCatalog();
  }

  public String getErrorMessage() {
    return errlog.toString();
  }

  public String getValidationMessage() {
    return errlog.toString();
  }

  public boolean hasFatalError() {
    return fatalError;
  }

  ////////////////////////////////////////////////////
  protected String name, version;
  protected CalendarDate expires;
  protected URI baseURI;
  protected List<Property> properties;
  protected List<Service> services;
  protected List<DatasetBuilder> datasetBuilders;

  public CatalogBuilder() {}

  public CatalogBuilder(Catalog from) {
    this.name       = from.getName();
    this.expires    = from.getExpires();
    this.baseURI    = from.getBaseURI();
    this.properties = from.getProperties();
    this.services   = from.getServices();  // ??
  }

  public void setName(String name) {
    this.name = name;
  }

  public void setBaseURI(URI baseURI) {
    this.baseURI = baseURI;
  }

  public void setExpires(CalendarDate expires) {
    this.expires = expires;
  }

  public void setVersion(String version) {
    this.version = version;
  }

  public void addProperty(Property p) {
    if (p == null) {
      return;
    }
    if (properties == null) {
      properties = new ArrayList<>();
    }
    if (!properties.contains(p)) {
      // O(n^2) but only done once
      properties.add(p);
    }
  }

  public void addService(Service s) {
    if (s == null) {
      return;
    }
    if (services == null) {
      services = new ArrayList<>();
      }
    if (!services.contains(s) && !containsService(s.getName())) {
      services.add(s);
    }
  }

  private boolean containsService(String name) {
    if (name == null) {
      return false;
    }
    if (services == null) {
      return false;
    }
    for (Service s : services) {
      if (name.equals(s.getName())) {
        return true;
      }
    }
    return false;
  }

  public void removeAnyService() {
    services = null;
  }

  public void addDataset(DatasetBuilder d) {
    if (d == null) {
      return;
    }
    if (datasetBuilders == null) {
      datasetBuilders = new ArrayList<>();
    }
    datasetBuilders.add(d);
  }

  ////////////////////////////////////////////////////////////////

  public Catalog makeCatalog() {
    setServices(getDatasets());
    Map<String, Object> flds = setFields();
    return new Catalog(baseURI, name, flds, datasetBuilders);
  }

  // pull services out of the datasets and into the catalog
  private void setServices(Iterable<DatasetBuilder> dsIter) {
    for (DatasetBuilder dsb : dsIter) {
      for (Service s : dsb.getServices()) {
        addService(s);
      }
      setServices(dsb.getDatasets()); // recurse
    }
  }

  protected Map<String, Object> setFields() {
    Map<String, Object> flds = new HashMap<>(10);

    if (expires    != null) { flds.put(Catalog.Expires,    expires); }
    if (version    != null) { flds.put(Catalog.Version,    version); }
    if (services   != null) { flds.put(Catalog.Services,   services); }
    if (properties != null) { flds.put(Dataset.Properties, properties); }

    return flds;
  }

  ///////////////////////////////////////////////////////////
  // get

  @Nullable
  public DatasetBuilder getTopDataset() {
    if (datasetBuilders == null) {
      return null;
    }
    return datasetBuilders.get(0);
  }

  public Iterable<DatasetBuilder> getDatasets() {
    if (datasetBuilders != null) {
      return datasetBuilders;
    }
    return new ArrayList<>(0);
  }

  public boolean hasService(String name) {
    if (services == null) {
      return false;
    }
    for (Service s : services) {
      if (s.getName().equalsIgnoreCase(name)) {
        return true;
      }
    }
    return false;
  }

  public boolean hasServiceInDataset(String name) {
    for (DatasetBuilder dsb : datasetBuilders) {
      for (Service s : dsb.getServices()) {
        if (s.getName().equalsIgnoreCase(name)) {
          return true;
        }
      }
    }
    return false;
  }

  /////////////////////////////////////////////////////////////////////
  // JDOM

  /* private void readXML(String fileName)  throws IOException {
    try {
      StAXStreamBuilder staxBuilder = new StAXStreamBuilder();
      XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
      XMLStreamReader xmlStreamReader = xmlInputFactory.createXMLStreamReader(new FileInputStream(fileName));
      Document jdomDoc = staxBuilder.build(xmlStreamReader);
      readCatalog(jdomDoc.getRootElement());

    } catch (Exception e) {
      errlog.format("failed to read catalog at '%s' err='%s'%n", fileName, e);
      logger.error("failed to read catalog at " + fileName, e);
      // e.printStackTrace();
      fatalError = true;
    }
  }  */

  private void readXML(String location) throws IOException {
     try {
       SAXBuilder saxBuilder = new SAXBuilder();
       Document jdomDoc = saxBuilder.build(location);
       readCatalog(jdomDoc.getRootElement());
     }
     catch (Exception e) {
       errlog.format("failed to read catalog at '%s' err='%s'%n", location, e);
       logger.error("failed to read catalog at {}, {}", location, e.toString());
       if (logger.isTraceEnabled()) {
         e.printStackTrace();
       }
       fatalError = true;
     }
   }

  private void readXML(URI uri) throws IOException {
    try {
      SAXBuilder saxBuilder = new SAXBuilder();
      Document jdomDoc = saxBuilder.build(uri.toURL());
      readCatalog(jdomDoc.getRootElement());
    }
    catch (Exception e) {
      errlog.format("failed to read catalog at '%s' err='%s'%n", uri.toString(), e);
      logger.error("failed to read catalog at {}, {}", uri.toString(), e.toString());
       if (logger.isTraceEnabled()) {
         e.printStackTrace();
       }
      fatalError = true;
    }
  }

  private void readXMLfromString(String catalogAsString) throws IOException {
    try {
      StringReader in = new StringReader(catalogAsString);
      SAXBuilder saxBuilder = new SAXBuilder();    // LOOK non-validating
      Document jdomDoc = saxBuilder.build(in);
      readCatalog(jdomDoc.getRootElement());
    }
    catch (Exception e) {
      errlog.format("failed to read catalogAsString err='%s'%n", e);
      logger.error("failed to read catalogAsString at {}, {}", baseURI.toString(), e.toString());
       if (logger.isTraceEnabled()) {
         e.printStackTrace();
       }
      fatalError = true;
    }
  }

  private void readXML(InputStream stream) throws IOException {
    try {
      SAXBuilder saxBuilder = new SAXBuilder();
      Document jdomDoc = saxBuilder.build(stream);
      readCatalog(jdomDoc.getRootElement());
    }
    catch (Exception e) {
      errlog.format("failed to read catalogAsString err='%s'%n", e);
      logger.error("failed to read catalogAsString at {}, {}", baseURI.toString(), e.toString());
       if (logger.isTraceEnabled()) {
         e.printStackTrace();
       }
      fatalError = true;
    }
  }

  /* <xsd:element name="catalog">
     <xsd:complexType>
       <xsd:sequence>
         <xsd:element ref="service" minOccurs="0" maxOccurs="unbounded"/>
         <xsd:element ref="property" minOccurs="0" maxOccurs="unbounded" />
         <xsd:element ref="dataset" minOccurs="1" maxOccurs="unbounded" />
       </xsd:sequence>

       <xsd:attribute name="name" type="xsd:string" />
       <xsd:attribute name="expires" type="dateType"/>
       <xsd:attribute name="version" type="xsd:token" default="1.0.2" />
     </xsd:complexType>
   </xsd:element>
   */
  private void readCatalog(Element catalogElem) {

    String name = catalogElem.getAttributeValue("name");
    String catSpecifiedBaseURL = catalogElem.getAttributeValue("base");   // LOOK what is this ??
    String expiresS = catalogElem.getAttributeValue("expires");
    String version = catalogElem.getAttributeValue("version");

    CalendarDate expires = null;
    if (expiresS != null) {
      try {
        expires = CalendarDateFormatter.isoStringToCalendarDate(null, expiresS);
      }
      catch (Exception e) {
        errlog.format("bad expires date '%s' err='%s'%n", expiresS, e.getMessage());
        logger.debug("bad expires date '{}' err='{}'%n", expiresS, e.getMessage());
      }
    }

    if (catSpecifiedBaseURL != null) {
      try {
        URI userSpecifiedBaseUri = new URI(catSpecifiedBaseURL);
        this.baseURI = userSpecifiedBaseUri;
      }
      catch (URISyntaxException e) {
        errlog.format("readCatalog(): bad catalog specified base URI='%s' %n", catSpecifiedBaseURL);
        logger.debug("bad catalog specified base URI='{}'");
      }
    }

    setName(name);
    setExpires(expires);
    setVersion(version);

    // read top-level services
    java.util.List<Element> sList = catalogElem.getChildren("service", Catalog.defNS);
    for (Element e : sList) {
      addService(readService(e));
    }

    // read top-level properties
    java.util.List<Element> pList = catalogElem.getChildren("property", Catalog.defNS);
    for (Element e : pList) {
      addProperty(readProperty(e));
    }

    // look for top-level dataset and catalogRefs elements (keep them in order)
    java.util.List<Element> allChildren = catalogElem.getChildren();
    for (Element e : allChildren) {
      if (e.getName().equals("dataset")) {
        addDataset(readDataset(null, e));
      }
      else if (e.getName().equals("catalogRef")) {
        addDataset(readCatalogRef(null, e));
      }
      else {
        addDataset(buildOtherDataset(null, e));
      }
    }
  }

  // for overridding
  protected DatasetBuilder buildOtherDataset(DatasetBuilder parent, Element dsElem) {
    return null;
  }

  /* <xsd:element name="access">
     <xsd:complexType>
       <xsd:sequence>
         <xsd:element ref="dataSize" minOccurs="0"/>   // whyd we do that ?
       </xsd:sequence>
       <xsd:attribute name="urlPath" type="xsd:token" use="required"/>
       <xsd:attribute name="serviceName" type="xsd:string"/>
       <xsd:attribute name="dataFormat" type="dataFormatTypes"/>
     </xsd:complexType>
   </xsd:element >
  */
  protected AccessBuilder readAccess(DatasetBuilder dataset, Element accessElem) {
    String urlPath = accessElem.getAttributeValue("urlPath");
    String serviceName = accessElem.getAttributeValue("serviceName");
    String dataFormat = accessElem.getAttributeValue("dataFormat");

    Service s = serviceMap.get(serviceName);
    if (s == null) {
      errlog.format("Cant find service name='%s'%n", serviceName);
      logger.debug("Can't find service name='{}'", serviceName);
    }
    return new AccessBuilder(dataset, urlPath, s, dataFormat, readDataSize(accessElem));
  }

  protected Property readProperty(Element s) {
    String name = s.getAttributeValue("name");
    String value = s.getAttributeValue("value");
    return new Property(name, value);
  }

  /* <xsd:element name="service">
    <xsd:complexType>
     <xsd:sequence>
       <xsd:element ref="property" minOccurs="0" maxOccurs="unbounded" />
       <xsd:element ref="service" minOccurs="0" maxOccurs="unbounded" />
     </xsd:sequence>

     <xsd:attribute name="name" type="xsd:string" use="required" />
     <xsd:attribute name="base" type="xsd:string" use="required" />
     <xsd:attribute name="serviceType" type="serviceTypes" use="required" />
     <xsd:attribute name="desc" type="xsd:string"/>
     <xsd:attribute name="accessType" type="xsd:string"/>
     <xsd:attribute name="suffix" type="xsd:string" />
    </xsd:complexType>
   </xsd:element>
   */
  protected Service readService(Element s) {
    String name = s.getAttributeValue("name");
    String typeS = s.getAttributeValue("serviceType");
    String serviceBase = s.getAttributeValue("base");
    String suffix = s.getAttributeValue("suffix");
    String desc = s.getAttributeValue("desc");
    String accessType = s.getAttributeValue("accessType");

    ServiceType type = ServiceType.getServiceTypeIgnoreCase(typeS);
    if (type == null) {
      errlog.format(" non-standard service type = '%s'%n", typeS);
      logger.debug(" non-standard service type = '{}'", typeS);
    }
    else {
      if (desc == null) {
        desc = type.getDescription();
      }
      if (accessType == null) {
        accessType = type.getAccessType();
      }
    }

    List<Property> properties = null;
    List<Element> propertyList = s.getChildren("property", Catalog.defNS);
    for (Element e : propertyList) {
      if (properties == null) {
        properties = new ArrayList<>();
      }
      properties.add(readProperty(e));
    }

    // nested services
    List<Service> services = null;
    java.util.List<Element> serviceList = s.getChildren("service", Catalog.defNS);
    for (Element e : serviceList) {
      if (services == null) {
        services = new ArrayList<>();
      }
      services.add(readService(e));
    }

    Service result = new Service(name, serviceBase, typeS, desc, suffix, services, properties, accessType);
    serviceMap.put(name, result);
    return result;
  }

  /*
    <xsd:element name="catalogRef" substitutionGroup="dataset">
    <xsd:complexType>
      <xsd:complexContent>
        <xsd:extension base="DatasetType">
          <xsd:attributeGroup ref="XLink"/>
          <xsd:attribute name="useRemoteCatalogService" type="xsd:boolean"/>
        </xsd:extension>
      </xsd:complexContent>
    </xsd:complexType>
  </xsd:element>
   */
  protected DatasetBuilder readCatalogRef(DatasetBuilder parent, Element catRefElem) {
    String title = catRefElem.getAttributeValue("title", Catalog.xlinkNS);
    if (title == null) {
      title = catRefElem.getAttributeValue("name");
    }
    String href = catRefElem.getAttributeValue("href", Catalog.xlinkNS);
    CatalogRefBuilder catRef = new CatalogRefBuilder(parent);
    readDatasetInfo( catRef, catRefElem);
    catRef.setTitle(title);
    catRef.setHref(href);
    return catRef;
  }

  /* <xsd:complexType name="DatasetType">
     <xsd:sequence>
       <xsd:group ref="threddsMetadataGroup" minOccurs="0" maxOccurs="unbounded" />

       <xsd:element ref="access" minOccurs="0" maxOccurs="unbounded"/>
       <xsd:element ref="ncml:netcdf" minOccurs="0"/>
       <xsd:element ref="dataset" minOccurs="0" maxOccurs="unbounded"/>
     </xsd:sequence>

     <xsd:attribute name="name" type="xsd:string" use="required"/>
     <xsd:attribute name="alias" type="xsd:token"/>
     <xsd:attribute name="authority" type="xsd:string"/> <!-- deprecated : use element -->
     <xsd:attribute name="collectionType" type="collectionTypes"/>
     <xsd:attribute name="dataType" type="dataTypes"/> <!-- deprecated : use element -->
     <xsd:attribute name="harvest" type="xsd:boolean"/>
     <xsd:attribute name="ID" type="xsd:token"/>
     <xsd:attribute name="resourceControl" type="xsd:string"/>

     <xsd:attribute name="serviceName" type="xsd:string" /> <!-- deprecated : use element -->
     <xsd:attribute name="urlPath" type="xsd:token" />
   </xsd:complexType> */
  protected DatasetBuilder readDataset(DatasetBuilder parent, Element dsElem) {

    DatasetBuilder dataset = new DatasetBuilder(parent);
    readDatasetInfo(dataset, dsElem);

    // look for access elements
    java.util.List<Element> aList = dsElem.getChildren("access", Catalog.defNS);
    for (Element e : aList) {
      dataset.addAccess(readAccess(dataset, e));
    }

    // look for nested dataset and catalogRefs elements (keep them in order)
    java.util.List<Element> allChildren = dsElem.getChildren();
    for (Element e : allChildren) {
      if (e.getName().equals("dataset")) {
        dataset.addDataset(readDataset(dataset, e));
      }
      else if (e.getName().equals("catalogRef")) {
        dataset.addDataset(readCatalogRef(dataset, e));
      }
      else {
        dataset.addDataset( buildOtherDataset(dataset, e));
      }
    }

    return dataset;
  }

  protected void readDatasetInfo(DatasetBuilder dataset, Element dsElem) {
    // read attributes
    String name = dsElem.getAttributeValue("name");
    if (name == null) {
      if (dsElem.getName().equals("catalogRef")) {
        dataset.setName("");
      }
      else {
        errlog.format(" ** warning: dataset must have a name = '%s'%n", dsElem);
        logger.debug(" ** warning: dataset must have a name = '{}'", dsElem);
      }
    }
    else {
      dataset.setName(name);
    }

    dataset.put( Dataset.Alias, dsElem.getAttributeValue("alias"));
    dataset.put( Dataset.Authority, dsElem.getAttributeValue("authority"));
    dataset.put( Dataset.CollectionType, dsElem.getAttributeValue("collectionType"));
    dataset.put( Dataset.Id, dsElem.getAttributeValue("ID"));
    dataset.putInheritedField( Dataset.RestrictAccess, dsElem.getAttributeValue("restrictAccess"));
    dataset.put( Dataset.ServiceName, dsElem.getAttributeValue("serviceName"));
    dataset.put( Dataset.UrlPath, dsElem.getAttributeValue("urlPath"));

    String dataTypeName = dsElem.getAttributeValue("dataType");
    dataset.put( Dataset.FeatureType, dataTypeName);
    if (dataTypeName != null) {
      FeatureType dataType = FeatureType.getType(dataTypeName.toUpperCase());
      if (dataType == null) {
        errlog.format(" ** warning: non-standard data type = '%s'%n", dataTypeName);
        logger.debug(" ** warning: non-standard data type = '{}'", dataTypeName);
      }
    }

    String harvest = dsElem.getAttributeValue("harvest");
    if (harvest != null && harvest.equalsIgnoreCase("true")) {
      dataset.put(Dataset.Harvest, Boolean.TRUE);
    }

    // catalog.addDatasetByID(dataset); // LOOK need to do immed for alias processing

    // read elements
    readThreddsMetadataGroup(dataset.flds, dataset, dsElem);
  }

  /*
   <!-- group of elements can be used in a dataset or in metadata elements -->
  <xsd:group name="threddsMetadataGroup">
    <xsd:choice>
      <xsd:element name="documentation" type="documentationType"/>
      <xsd:element ref="metadata"/>
      <xsd:element ref="property"/>

      <xsd:element ref="contributor"/>
      <xsd:element name="creator" type="sourceType"/>
      <xsd:element name="date" type="dateTypeFormatted"/>
      <xsd:element name="keyword" type="controlledVocabulary"/>
      <xsd:element name="project" type="controlledVocabulary"/>
      <xsd:element name="publisher" type="sourceType"/>

      <xsd:element ref="geospatialCoverage"/>
      <xsd:element name="timeCoverage" type="timeCoverageType"/>
      <xsd:element ref="variables"/>
      <xsd:element ref="variableMap"/>

      <xsd:element name="dataType" type="dataTypes"/>
      <xsd:element name="dataFormat" type="dataFormatTypes"/>
      <xsd:element name="serviceName" type="xsd:string"/>
      <xsd:element name="authority" type="xsd:string"/>
      <xsd:element ref="dataSize"/>
    </xsd:choice>
  </xsd:group>
   */
  protected void readThreddsMetadataGroup(Map<String,Object> flds, DatasetBuilder dataset, Element parent) {
    List<Element> list;

    // look for creators - kind of a Source
    list = parent.getChildren("creator", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Creators, readSource(e));
    }

    // look for contributors
    list = parent.getChildren("contributor", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Contributors, readContributor(e));
    }

    // look for dates
    list = parent.getChildren("date", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Dates, readDate(e, null));
    }

    // look for documentation
    list = parent.getChildren("documentation", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Documentation, readDocumentation(e));
    }

    // look for keywords - kind of a controlled vocabulary
    list = parent.getChildren("keyword", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Keywords, readControlledVocabulary(e));
    }

    // look for metadata elements
    list = parent.getChildren("metadata", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.MetadataOther, readMetadata(flds, dataset, e));
    }

    // look for projects - kind of a controlled vocabulary
    list = parent.getChildren("project", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Projects, readControlledVocabulary(e));
    }

    // look for properties
    list = parent.getChildren("property", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Properties, readProperty(e));
    }

    // look for publishers - kind of a Source
    list = parent.getChildren("publisher", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.Publishers, readSource(e));
    }

    // look for variables
    list = parent.getChildren("variables", Catalog.defNS);
    for (Element e : list) {
      DatasetBuilder.addToList(flds, Dataset.VariableGroups, readVariables(e));
    }

    // can only be one each of these kinds
    ThreddsMetadata.GeospatialCoverage gc = readGeospatialCoverage(parent.getChild("geospatialCoverage", Catalog.defNS));
    if (gc != null) {
      flds.put(Dataset.GeospatialCoverage, gc);
    }

    DateRange tc = readTimeCoverage(parent.getChild("timeCoverage", Catalog.defNS));
    if (tc != null) {
      flds.put(Dataset.TimeCoverage, tc);
    }

    Element serviceNameElem = parent.getChild("serviceName", Catalog.defNS);
    if (serviceNameElem != null) {
      flds.put(Dataset.ServiceName, serviceNameElem.getText());
    }

    Element authElem = parent.getChild("authority", Catalog.defNS);
    if (authElem != null) {
      flds.put(Dataset.Authority, authElem.getText());
    }

    Element dataTypeElem = parent.getChild("dataType", Catalog.defNS);
    if (dataTypeElem != null) {
      String dataTypeName = dataTypeElem.getText();
      flds.put(Dataset.FeatureType, dataTypeName);
      if ((dataTypeName != null) && (dataTypeName.length() > 0)) {
        FeatureType dataType = FeatureType.getType(dataTypeName.toUpperCase());
        if (dataType == null) {
          errlog.format(" ** warning: non-standard feature type = '%s'%n", dataTypeName);
          logger.debug(" ** warning: non-standard feature type = '{}'", dataTypeName);
        }
      }
    }

    Element dataFormatElem = parent.getChild("dataFormat", Catalog.defNS);
    if (dataFormatElem != null) {
      String dataFormatTypeName = dataFormatElem.getText();
      if ((dataFormatTypeName != null) && (dataFormatTypeName.length() > 0)) {
        DataFormatType dataFormatType = DataFormatType.getType(dataFormatTypeName);
        if (dataFormatType == null) {
          errlog.format(" ** warning: non-standard dataFormat type = '%s'%n", dataFormatTypeName);
          logger.debug(" ** warning: non-standard dataFormat type = '{}'", dataFormatTypeName);
        }
        flds.put(Dataset.DataFormatType, dataFormatTypeName);
      }
    }

    long size = readDataSize(parent);
    if (size > 0) {
      flds.put(Dataset.DataSize, size);
    }

    // LOOK: we seem to have put a variableMap element not contained by <variables>
    ThreddsMetadata.UriResolved mapUri = readUri(parent.getChild("variableMap", Catalog.defNS), "variableMap");
    if (mapUri != null) {
      flds.put(Dataset.VariableMapLinkURI, mapUri);
    }
  }

  protected ThreddsMetadata.Contributor readContributor(Element elem) {
    if (elem == null) {
      return null;
    }
    return new ThreddsMetadata.Contributor(elem.getText(), elem.getAttributeValue("role"));
  }

  protected long readDataSize(Element parent) {
    Element elem = parent.getChild("dataSize", Catalog.defNS);
    if (elem == null) {
      return -1;
    }

    double size;
    String sizeS = elem.getText();
    try {
      size = Double.parseDouble(sizeS);
    }
    catch (NumberFormatException e) {
      errlog.format(" ** Parse error: Bad double format in size element = '%s'%n", sizeS);
      logger.debug(" ** Parse error: Bad double format in size element = '{}'", sizeS);
      return -1;
    }

    String units = elem.getAttributeValue("units");
    char c = Character.toUpperCase(units.charAt(0));
    if (c == 'K') {
      size *= 1000;
    }
    else if (c == 'M') {
      size *= 1000 * 1000;
    }
    else if (c == 'G') {
      size *= 1000 * 1000 * 1000;
    }
    else if (c == 'T') {
      size *= 1000.0 * 1000 * 1000 * 1000;
    }
    else if (c == 'P') {
      size *= 1000.0 * 1000 * 1000 * 1000 * 1000;
    }
    return (long) size;
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  protected Documentation readDocumentation(Element s) {
    String href = s.getAttributeValue("href", Catalog.xlinkNS);
    String title = s.getAttributeValue("title", Catalog.xlinkNS);
    String type = s.getAttributeValue("type"); // not XLink type
    String content = s.getTextNormalize();

    URI uri = null;
    if (href != null) {
      try {
        uri = Catalog.resolveUri(baseURI, href);
      }
      catch (Exception e) {
        errlog.format(" ** Invalid documentation href = '%s' err='%s'%n", href, e.getMessage());
        logger.debug(" ** Invalid documentation href = '{}' err='{}'", href, e.getMessage());
      }
    }

    return new Documentation(href, uri, title, type, content);
  }

  protected double readDouble(Element elem) {
    if (elem == null) {
      return Double.NaN;
    }
    String text = elem.getText();
    try {
      return Double.parseDouble(text);
    }
    catch (NumberFormatException e) {
      errlog.format(" ** Parse error: Bad double format = '%s'%n", text);
      logger.debug(" ** Parse error: Bad double format = '{}'", text);
      return Double.NaN;
    }
  }

  /*
    <xsd:element name="geospatialCoverage">
    <xsd:complexType>
      <xsd:sequence>
        <xsd:element name="northsouth" type="spatialRange" minOccurs="0"/>
        <xsd:element name="eastwest" type="spatialRange" minOccurs="0"/>
        <xsd:element name="updown" type="spatialRange" minOccurs="0"/>
        <xsd:element name="name" type="controlledVocabulary" minOccurs="0" maxOccurs="unbounded"/>
      </xsd:sequence>

      <xsd:attribute name="zpositive" type="upOrDown" default="up"/>
    </xsd:complexType>
  </xsd:element>

  <xsd:complexType name="spatialRange">
    <xsd:sequence>
      <xsd:element name="start" type="xsd:double"/>
      <xsd:element name="size" type="xsd:double"/>
      <xsd:element name="resolution" type="xsd:double" minOccurs="0"/>
      <xsd:element name="units" type="xsd:string" minOccurs="0"/>
    </xsd:sequence>
  </xsd:complexType>

  <xsd:simpleType name="upOrDown">
    <xsd:restriction base="xsd:token">
      <xsd:enumeration value="up"/>
      <xsd:enumeration value="down"/>
    </xsd:restriction>
  </xsd:simpleType>
   */
  protected ThreddsMetadata.GeospatialCoverage readGeospatialCoverage(Element gcElem) {
    if (gcElem == null) {
      return null;
    }

    String zpositive = gcElem.getAttributeValue("zpositive");

    ThreddsMetadata.GeospatialRange northsouth = readGeospatialRange(gcElem.getChild("northsouth", Catalog.defNS), CDM.LAT_UNITS);
    ThreddsMetadata.GeospatialRange eastwest = readGeospatialRange(gcElem.getChild("eastwest", Catalog.defNS), CDM.LON_UNITS);
    ThreddsMetadata.GeospatialRange updown = readGeospatialRange(gcElem.getChild("updown", Catalog.defNS), "m");

    // look for names
    List<ThreddsMetadata.Vocab> names = new ArrayList<>();
    java.util.List<Element> list = gcElem.getChildren("name", Catalog.defNS);
    for (Element e : list) {
      ThreddsMetadata.Vocab name = readControlledVocabulary(e);
      names.add(name);
    }

    return new ThreddsMetadata.GeospatialCoverage(eastwest, northsouth, updown, names, zpositive);
  }

  protected ThreddsMetadata.GeospatialRange readGeospatialRange(Element spElem, String defUnits) {
    if (spElem == null) {
      return null;
    }

    double start = readDouble(spElem.getChild("start", Catalog.defNS));
    double size = readDouble(spElem.getChild("size", Catalog.defNS));
    double resolution = readDouble(spElem.getChild("resolution", Catalog.defNS));

    String units = spElem.getChildText("units", Catalog.defNS);
    if (units == null) {
      units = defUnits;
    }

    return new ThreddsMetadata.GeospatialRange(start, size, resolution, units);
  }

  /*
    <xsd:element name="metadata">
    <xsd:complexType>
      <xsd:choice>
        <xsd:group ref="threddsMetadataGroup" minOccurs="0" maxOccurs="unbounded"/>
        <xsd:any namespace="##other" minOccurs="0" maxOccurs="unbounded" processContents="lax"/>
      </xsd:choice>

      <xsd:attribute name="inherited" type="xsd:boolean" default="false"/>
      <xsd:attribute name="metadataType" type="metadataTypeEnum"/>
      <xsd:attributeGroup ref="XLink"/>
    </xsd:complexType>
  </xsd:element>
   */
  protected ThreddsMetadata.MetadataOther readMetadata(Map<String,Object> flds, DatasetBuilder dataset, Element mdataElement) {
    // there are 6 cases to deal with: threddsNamespace vs not & inline vs Xlink & (if thredds) inherited or not
    Namespace namespace;
    List inlineElements = mdataElement.getChildren();
    if (inlineElements.size() > 0) {
      // look at the namespace of the children, if they exist
      namespace = ((Element) inlineElements.get(0)).getNamespace();
    }
    else {
      namespace = mdataElement.getNamespace(); // will be thredds
    }

    String mtype = mdataElement.getAttributeValue("metadataType");
    String href = mdataElement.getAttributeValue("href", Catalog.xlinkNS);
    String title = mdataElement.getAttributeValue("title", Catalog.xlinkNS);
    String inheritedS = mdataElement.getAttributeValue("inherited");
    boolean inherited = (inheritedS != null) && inheritedS.equalsIgnoreCase("true");

    boolean isThreddsNamespace = ((mtype == null) || mtype.equalsIgnoreCase("THREDDS")) && namespace.getURI().equals(Catalog.CATALOG_NAMESPACE_10);

    // the case where its not ThreddsMetadata
    if (!isThreddsNamespace) {
      if (inlineElements.size() > 0) {
        // just hold onto the jdom elements as the "content"
         return new ThreddsMetadata.MetadataOther( mtype, namespace.getURI(), namespace.getPrefix(), inherited, mdataElement);

      }
      else {
        // otherwise it must be an Xlink
        return new ThreddsMetadata.MetadataOther(href, title, mtype, namespace.getURI(), namespace.getPrefix(), inherited);
      }
    }

    // the case where its ThreddsMetadata
    Map<String,Object> useFlds;
    if (inherited) {
      // the case where its inherited ThreddsMetadata: gonna put stuff in the tmi.
      ThreddsMetadata tmi = (ThreddsMetadata) dataset.get(Dataset.ThreddsMetadataInheritable);
      if (tmi == null) {
        tmi = new ThreddsMetadata();
        dataset.put(Dataset.ThreddsMetadataInheritable, tmi);
      }
      useFlds = tmi.getFlds();

    }
    else {
      // the case where its non-inherited ThreddsMetadata: gonna put stuff directly into the dataset
      useFlds = flds;
    }
    readThreddsMetadataGroup(useFlds, dataset, mdataElement);

    // also need to capture any XLinks. see http://www.unidata.ucar.edu/software/thredds/v4.6/tds/catalog/InvCatalogSpec.html#metadataElement
    // in this case we just suck it in as if it was inline
    if (href != null) {
      try {
        URI xlinkUri = Catalog.resolveUri(baseURI, href);
        Element remoteMdata = readMetadataFromUrl(xlinkUri);
        return readMetadata(useFlds, dataset, remoteMdata);
      }
      catch (Exception ioe) {
        errlog.format("Cant read in referenced metadata %s err=%s%n", href, ioe.getMessage());
        logger.debug("Can't read in referenced metadata {} err={}}", href, ioe.getMessage());
      }

    }
    return null;  // ThreddsMetadata.MetadataOther was directly added
  }

  private Element readMetadataFromUrl(java.net.URI uri) throws java.io.IOException {
    SAXBuilder saxBuilder = new SAXBuilder();
    Document doc;
    try {
      doc = saxBuilder.build(uri.toURL());
    }
    catch (Exception e) {
      throw new IOException(e.getMessage());
    }
    return doc.getRootElement();
  }

  /*
    <xsd:complexType name="sourceType">
    <xsd:sequence>
      <xsd:element name="name" type="controlledVocabulary"/>

      <xsd:element name="contact">
        <xsd:complexType>
          <xsd:attribute name="email" type="xsd:string" use="required"/>
          <xsd:attribute name="url" type="xsd:anyURI"/>
        </xsd:complexType>
      </xsd:element>

    </xsd:sequence>
  </xsd:complexType>
   */
  protected ThreddsMetadata.Source readSource(Element elem) {
    if (elem == null) {
      return null;
    }
    ThreddsMetadata.Vocab name = readControlledVocabulary(elem.getChild("name", Catalog.defNS));
    Element contact = elem.getChild("contact", Catalog.defNS);
    if (contact == null) {
      errlog.format(" ** Parse error: Missing contact element in = '%s'%n", elem.getName());
      logger.debug(" ** Parse error: Missing contact element in = '{}'", elem.getName());
      return null;
    }
    return new ThreddsMetadata.Source(name, contact.getAttributeValue("url"), contact.getAttributeValue("email"));
  }

  /*
    <xsd:complexType name="timeCoverageType">
    <xsd:sequence>
      <xsd:attribute name="calendar" type="xsd:string"/>
      <xsd:choice minOccurs="2" maxOccurs="3">
        <xsd:element name="start" type="dateTypeFormatted"/>
        <xsd:element name="end" type="dateTypeFormatted"/>
        <xsd:element name="duration" type="duration"/>
      </xsd:choice>
      <xsd:element name="resolution" type="duration" minOccurs="0"/>
    </xsd:sequence>
  </xsd:complexType>

  <!-- may be a dateType or have a format attribute  -->
  <xsd:complexType name="dateTypeFormatted">
    <xsd:simpleContent>
      <xsd:extension base="dateType">
        <xsd:attribute name="format" type="xsd:string"/>
        <!-- follow java.text.SimpleDateFormat -->
        <xsd:attribute name="type" type="dateEnumTypes"/>
      </xsd:extension>
    </xsd:simpleContent>
  </xsd:complexType>

  <!-- may be a built in date or dateTIme, or a udunit encoded string -->
  <xsd:simpleType name="dateType">
    <xsd:union memberTypes="xsd:date xsd:dateTime udunitDate">
      <xsd:simpleType>
        <xsd:restriction base="xsd:token">
          <xsd:enumeration value="present"/>
        </xsd:restriction>
      </xsd:simpleType>
    </xsd:union>
  </xsd:simpleType>

  <xsd:simpleType name="udunitDate">
    <xsd:restriction base="xsd:string">
      <xsd:annotation>
        <xsd:documentation>Must conform to complete udunits date string, eg "20 days since 1991-01-01"</xsd:documentation>
      </xsd:annotation>
    </xsd:restriction>
  </xsd:simpleType>

  <xsd:simpleType name="duration">
    <xsd:union memberTypes="xsd:duration udunitDuration"/>
  </xsd:simpleType>

  <xsd:simpleType name="udunitDuration">
    <xsd:restriction base="xsd:string">
      <xsd:annotation>
        <xsd:documentation>Must conform to udunits time duration, eg "20.1 hours"</xsd:documentation>
      </xsd:annotation>
    </xsd:restriction>
  </xsd:simpleType>
   */
  protected DateRange readTimeCoverage(Element tElem) {
    if (tElem == null) {
      return null;
    }

    Calendar calendar = readCalendar(tElem.getAttributeValue("calendar"));
    DateType start = readDate(tElem.getChild("start", Catalog.defNS), calendar);
    DateType end = readDate(tElem.getChild("end", Catalog.defNS), calendar);

    TimeDuration duration = readDuration(tElem.getChild("duration", Catalog.defNS));
    TimeDuration resolution = readDuration(tElem.getChild("resolution", Catalog.defNS));

    try {
      return new DateRange(start, end, duration, resolution);
    }
    catch (java.lang.IllegalArgumentException e) {
      errlog.format(" ** warning: TimeCoverage error ='%s'%n", e.getMessage());
      logger.debug(" ** warning: TimeCoverage error ='{}'", e.getMessage());
      return null;
    }
  }

  protected Calendar readCalendar(String calendarAttribValue) {
    if (calendarAttribValue == null) {
      return Calendar.getDefault();
    }

    Calendar calendar = Calendar.get(calendarAttribValue);
    if (calendar == null) {
      errlog.format(" ** Parse error: Bad calendar name = '%s'%n", calendarAttribValue);
      logger.debug(" ** Parse error: Bad calendar name = '{}}'", calendarAttribValue);
      return Calendar.getDefault();
    }

    return calendar;
  }

  protected DateType readDate(Element elem, Calendar calendar) {
    if (elem == null) {
      return null;
    }
    String format = elem.getAttributeValue("format");
    String type = elem.getAttributeValue("type");
    return makeDateType(elem.getText(), format, type, calendar);
  }

  protected DateType makeDateType(String text, String format, String type, Calendar calendar) {
    if (text == null) {
      return null;
    }
    try {
      return new DateType(text, format, type, calendar);
    }
    catch (java.text.ParseException e) {
      errlog.format(" ** Parse error: Bad date format = '%s'%n", text);
      return null;
    }
  }

  protected TimeDuration readDuration(Element elem) {
    if (elem == null) {
      return null;
    }
    String text = null;
    try {
      text = elem.getText();
      return new TimeDuration(text);
    }
    catch (java.text.ParseException e) {
      errlog.format(" ** Parse error: Bad duration format = '%s'%n", text);
      return null;
    }
  }

  /*
    <xsd:element name="variables">
    <xsd:complexType>
      <xsd:choice>
        <xsd:element ref="variable" minOccurs="0" maxOccurs="unbounded"/>
        <xsd:element ref="variableMap" minOccurs="0"/>
      </xsd:choice>
      <xsd:attribute name="vocabulary" type="variableNameVocabulary" use="optional"/>
      <xsd:attributeGroup ref="XLink"/>
    </xsd:complexType>
  </xsd:element>

  <xsd:element name="variable">
    <xsd:complexType mixed="true">
      <xsd:attribute name="name" type="xsd:string" use="required"/>
      <xsd:attribute name="vocabulary_name" type="xsd:string" use="optional"/>
      <xsd:attribute name="vocabulary_id" type="xsd:string" use="optional"/>
      <xsd:attribute name="units" type="xsd:string"/>
    </xsd:complexType>
  </xsd:element>

  <xsd:element name="variableMap">
    <xsd:complexType>
      <xsd:attributeGroup ref="XLink"/>
    </xsd:complexType>
  </xsd:element>

  <xsd:simpleType name="variableNameVocabulary">
    <xsd:union memberTypes="xsd:token">
      <xsd:simpleType>
        <xsd:restriction base="xsd:token">
          <xsd:enumeration value="CF-1.0"/>
          <xsd:enumeration value="DIF"/>
          <xsd:enumeration value="GRIB-1"/>
          <xsd:enumeration value="GRIB-2"/>
        </xsd:restriction>
      </xsd:simpleType>
    </xsd:union>
  </xsd:simpleType>
   */
  protected ThreddsMetadata.VariableGroup readVariables( Element varsElem) {
    if (varsElem == null) {
      return null;
    }

    String vocab = varsElem.getAttributeValue("vocabulary");
    ThreddsMetadata.UriResolved variableVocabUri = readUri(varsElem, "Variables vocabulary");

    java.util.List<Element> vlist = varsElem.getChildren("variable", Catalog.defNS);
    ThreddsMetadata.UriResolved variableMap = readUri(varsElem.getChild("variableMap", Catalog.defNS), "Variables Map");
    if ((variableMap != null) && vlist.size() > 0) { // cant do both
      errlog.format(" ** Catalog error: cant have variableMap and variable in same element '%s'%n", varsElem);
    }

    List<ThreddsMetadata.Variable> variables = new ArrayList<>();
    for (Element e : vlist) {
      variables.add(readVariable(e));
    }

    return new ThreddsMetadata.VariableGroup(vocab, variableVocabUri, variableMap, variables);
  }

  static public ThreddsMetadata.Variable readVariable(Element varElem) {
     if (varElem == null) {
       return null;
     }

     String name = varElem.getAttributeValue("name");
     String desc = varElem.getText();
     String vocabulary_name = varElem.getAttributeValue("vocabulary_name");
     String units = varElem.getAttributeValue("units");
     String id = varElem.getAttributeValue("vocabulary_id");

     return new ThreddsMetadata.Variable(name, desc, vocabulary_name, units, id);
   }

  protected ThreddsMetadata.Vocab readControlledVocabulary(Element elem) {
    if (elem == null) {
      return null;
    }
    return new ThreddsMetadata.Vocab(elem.getText(), elem.getAttributeValue("vocabulary"));
  }

  private ThreddsMetadata.UriResolved readUri(Element elemWithHref, String what) {
    if (elemWithHref == null) {
      return null;
    }
    String mapHref = elemWithHref.getAttributeValue("href", Catalog.xlinkNS);
    if (mapHref == null) {
      return null;
    }

    try {
      String mapUri = URLnaming.resolve(baseURI.toString(), mapHref);
      return new ThreddsMetadata.UriResolved(mapHref, new URI(mapUri));
    }
    catch (Exception e) {
      errlog.format(" ** Invalid %s URI= '%s' err='%s'%n", what, mapHref, e.getMessage());
      return null;
    }
  }
}