/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.servers;

import opendap.dap.*;

import java.io.*;

/**
 * Holds a OPeNDAP Server List value.
 *
 * @author ndp
 * @version $Revision: 15901 $
 * @see BaseType
 */
public abstract class SDList extends DList implements ServerMethods, RelOps {

    private static final boolean _Debug = false;

    private boolean Synthesized;
    private boolean ReadMe;

    /**
     * Constructs a new SDList.
     */
    public SDList() {
        super();
        Synthesized = false;
        ReadMe = false;
    }

    /**
     * Constructs a new SDList with name n.
     *
     * @param n the name of the variable.
     */
    public SDList(String n) {
        super(n);
        Synthesized = false;
        ReadMe = false;
    }

    /**
     * Write the variable's declaration in a C-style syntax. This
     * function is used to create textual representation of the Data
     * Descriptor Structure (DDS).  See The OPeNDAP User Manual for
     * information about this structure.
     *
     * @param os          The PrintWriter on which to print the
     *                    declaration.
     * @param space       Each line of the declaration will begin with the
     *                    characters in this string.  Usually used for leading spaces.
     * @param print_semi  a boolean value indicating whether to print a
     *                    semicolon at the end of the declaration.
     * @param constrained a boolean value indicating whether to print
     *                    the declartion dependent on the projection information. This
     *                    is only used by Server side code.
     * @see BaseType#printDecl(PrintWriter, String, boolean, boolean)
     */
    public void printDecl(PrintWriter os, String space,
                          boolean print_semi, boolean constrained) {
        if (constrained && !isProject())
            return;

        // BEWARE! Since printDecl()is (multiple) overloaded in BaseType
        // and all of the different signatures of printDecl() in BaseType
        // lead to one signature, we must be careful to override that
        // SAME signature here. That way all calls to printDecl() for
        // this object lead to this implementation.

        // Also, since printDecl()is (multiple) overloaded in BaseType
        // and all of the different signatures of printDecl() in BaseType
        // lead to the signature we are overriding here, we MUST call
        // the printDecl with the SAME signature THROUGH the super class
        // reference (assuming we want the super class functionality). If
        // we do otherwise, we will create an infinte call loop. OOPS!

        super.printDecl(os, space, print_semi, constrained);
    }

    /**
     * Prints the value of the variable, with its declaration.  This
     * function is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *

     *
 Important Note

     * This method overrides the BaseType method of the same name and
     * type signature and it significantly changes the behavior for all versions
     * of printVal() for this type:
     *  All the various versions of printVal() will only
     * print a value, or a value with declaration, if the variable is
     * in the projection.
     *

     *
In other words, if a call to
     * isProject() for a particular variable returns
     * true then printVal() will print a value
     * (or a declaration and a value).
     *

     *
If isProject() for a particular variable returns
     * false then printVal() is basically a No-Op.
     *

     *

     *
     * @param os           the PrintWriter on which to print the value.
     * @param space        this value is passed to the printDecl method,
     *                     and controls the leading spaces of the output.
     * @param print_decl_p a boolean value controlling whether the
     *                     variable declaration is printed as well as the value.
     * @see BaseType#printVal(PrintWriter, String, boolean)
     * @see ServerMethods#isProject()
     */
    public void printVal(PrintWriter os, String space, boolean print_decl_p) {
        if (!isProject())
            return;

        PrimitiveVector pv = getPrimitiveVector();

        if (pv instanceof BaseTypePrimitiveVector) {

            BaseTypePrimitiveVector vals = (BaseTypePrimitiveVector) pv;

            if (print_decl_p) {
                printDecl(os, space, false, true);
                os.print(" = ");
            }

            os.print("{ ");
            //vals.printVal(os, "");

            ServerMethods sm;
            int len = vals.getLength();
            for (int i = 0; i < len - 1; i++) {
                sm = (ServerMethods) vals.getValue(i);
                if (sm.isProject()) {
                    ((BaseType) sm).printVal(os, "", false);
                    os.print(", ");
                }
            }
            // print last value, if any, without trailing comma
            if (len > 0) {
                sm = (ServerMethods) vals.getValue(len - 1);
                if (sm.isProject()) {
                    ((BaseType) sm).printVal(os, "", false);
                }
            }

            os.print("}");

            if (print_decl_p)
                os.println(";");
        } else {
            super.printVal(os, space, print_decl_p);
        }
    }

    // --------------- Projection Interface

    /**
     * Set the state of this variable's projection. true means
     * that this variable is part of the current projection as defined by
     * the current constraint expression, otherwise the current projection
     * for this variable should be false.
     *
     * @param state true if the variable is part of the current
     *              projection, false otherwise.
     * @param all   This parameter has no effect for this type of variable.
     * @see CEEvaluator
     */
    @Override
    public void setProject(boolean state, boolean all) {
        setProjected(state);
        PrimitiveVector vals = getPrimitiveVector();
        ((ServerMethods) (vals.getTemplate())).setProject(state, all);

//		if(vals instanceof BaseTypePrimitiveVector){
//			System.out.println("Setting Projecttion for List member: "+
//			((BaseType)((BaseTypePrimitiveVector)vals).getTemplate()).getTypeName() + " " +
//			((BaseType)((BaseTypePrimitiveVector)vals).getTemplate()).getName());
//			((ServerMethods)((BaseTypePrimitiveVector)vals).getTemplate()).setProject(state);
//		}

    }

// --------------- RelOps Interface

    /**
     * The RelOps interface defines how each type responds to relational
     * operators. Most (all?) types will not have sensible responses to all of
     * the relational operators (e.g. DList won't know how to match a regular
     * expression but DString will). For those operators that are nonsensical a
     * class should throw InvalidOperatorException.
     */

    public boolean equal(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("Equals (=) operator does not work with the type SDList!");
    }

    public boolean not_equal(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("Not Equals (!=) operator does not work with the type SDList!");
    }

    public boolean greater(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("Greater Than (>)operator does not work with the type SDList!");
    }

    public boolean greater_eql(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("GreaterThan or equals (<=) operator does not work with the type SDList!");
    }

    public boolean less(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("LessThan (<) operator does not work with the type SDList!");
    }

    public boolean less_eql(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("LessThan oe equals (<=) operator does not work with the type SDList!");
    }

    public boolean regexp(BaseType bt) throws InvalidOperatorException, RegExpException, SBHException {
        throw new InvalidOperatorException("Regular Expression's don't work with the type SDList!");
    }

// --------------- FileIO Interface

    /**
     * Set the Synthesized property.
     *
     * @param state If true then the variable is considered a
     *              synthetic variable and no part of OPeNDAP will ever try to read it from a
     *              file, otherwise if false the variable is considered a
     *              normal variable whose value should be read using the
     *              read() method. By default this property is false.
     * @see #isSynthesized()
     * @see #read(String, Object)
     */
    public void setSynthesized(boolean state) {
        Synthesized = state;
    }

    /**
     * Get the value of the Synthesized property.
     *
     * @return true if this is a synthetic variable,
     *         false otherwise.
     */
    public boolean isSynthesized() {
        return (Synthesized);
    }

    /**
     * Set the Read property. A normal variable is read using the
     * read() method. Once read the Read property is
     * true. Use this function to manually set the property
     * value. By default this property is false.
     *
     * @param state true if the variable has been read,
     *              false otherwise.
     * @see #isRead()
     * @see #read(String, Object)
     */
    public void setRead(boolean state) {
        ReadMe = state;
    }

    /**
     * Get the value of the Read property.
     *
     * @return true if the variable has been read,
     *         false otherwise.
     * @see #read(String, Object)
     * @see #setRead (boolean)
     */
    public boolean isRead() {
        return (ReadMe);
    }

    /**
     * Read a value from the named dataset for this variable.
     *
Caution:

     * When reading Lists of sequences (children of DSequence) it is crucial
     * that it be handled with great care. Sequences have been implemented so that
     * only one instance (or row if you will) is retained in memory at a given time.
     * In order to correctly read a List of Sequences the read() method for the
     * List must create an instance of the Sequence for each member of the List, typically
     * by repeatedly cloning the template variable in the PrimitiveVector. The important next
     * step is: DO NOT attempt to read any data into the sequences from within the read()
     * method of the List. The sequence's data will get read, and constraint expressions
     * applied when the serialze() method of the List calls the serialize() method
     * of the Sequence, which in turn calls the read method of the Sequence. Good Luck!
     *
     * @param datasetName String identifying the file or other data store
     *                    from which to read a vaue for this variable.
     * @param specialO    This Object is a goody that is used by Server implementations
     *                    to deliver important, and as yet unknown, stuff to the read method. If you
     *                    don't need it, make it a null.
     * @return true if more data remains to be read, otherwise
     *         false. This is an abtsract method that must be implemented
     *         as part of the installation/localization of a OPeNDAP server.
     * @throws IOException
     * @throws EOFException
     */

    public abstract boolean read(String datasetName, Object specialO) throws NoSuchVariableException, IOException, EOFException;

    /**
     *

     * Server-side serialization for OPeNDAP variables (sub-classes of
     * BaseType).
     * This does not send the entire class as the Java Serializable
     * interface does, rather it sends only the binary data values. Other software
     * is responsible for sending variable type information (see DDS).
     *

     * Writes data to a DataOutputStream. This method is used
     * on the server side of the OPeNDAP client/server connection, and possibly
     * by GUI clients which need to download OPeNDAP data, manipulate it, and
     * then re-save it as a binary file.
     *

     *
Caution:

     * When serializing lists of sequences (children of DSequence) it is crucial
     * that it be handled with great care. Sequences have been implemented so that
     * only one instance (or row if you will) is retained in memory at a given time.
     * In order to correctly serialize a list of sequences the read() method for the
     * List must create an instance of the sequence for each member of the list, typically
     * by repeatedly cloning the template variable in the PrimitiveVector. The important next
     * step is to NOT attempt to read any data into the sequences from within the read()
     * method of the List. The sequence's data will get read, and constraint expressions
     * applied when the serialze() method of the List calls the serialize() method
     * of the sequence. Good Luck!
     *
     * @param sink a DataOutputStream to write to.
     * @throws IOException thrown on any OutputStream exception.
     * @see BaseType
     * @see DDS
     * @see ServerDDS
     */
    public void serialize(String dataset, DataOutputStream sink, CEEvaluator ce, Object specialO)
            throws NoSuchVariableException, DAP2ServerSideException, IOException {

        PrimitiveVector vals = getPrimitiveVector();

        if (_Debug) System.out.println("\nSTARTED.............SDList.serialize(" + dataset + ")");

        if (!isRead())
            read(dataset, specialO);

        if (vals.getTemplate() instanceof DSequence || ce.evalClauses(specialO)) {

            // Because arrays of primitive types (ie int32, float32, byte, etc) are
            // handled in the C++ core using the XDR package we must write the
            // length twice for those types. For BaseType vectors, we should write
            // it only once. This is in effect a work around for a bug in the C++
            // core as the C++ core does not consume 2 length values for thge
            // BaseType vectors. Bummer...
            int length = vals.getLength();
            sink.writeInt(length);

            // Gotta check for this to make sure that DConstructor types
            // (Especially SDSequence) get handled correctly!!!
            if (vals instanceof BaseTypePrimitiveVector) {
                for (int i = 0; i < length; i++) {
                    ServerMethods sm = (ServerMethods) ((BaseTypePrimitiveVector) vals).getValue(i);
                    sm.serialize(dataset, sink, ce, specialO);
                }
            } else {

                // because both XDR and OPeNDAP read the length, we must write it twice
                sink.writeInt(length);
                vals.externalize(sink);
            }

        }
        if (_Debug) System.out.println("\nFINISHED............SDList.serialize(" + dataset + ") ");

    }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.bufr.tables;

/*
 * BufrRead mnemonic.java  1.0  05/09/2008
 * @author  Robb Kambic
 * @version 1.0
 */

import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.bufr.*;
import ucar.nc2.util.IO;
import ucar.unidata.util.StringUtil2;

import java.io.*;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.lang.Integer;

/**
 * A class that reads a  mnemonic table. It doesn't include X < 48 and Y < 192 type of
 * descriptors because they are already stored in the latest WMO tables.
 */

public class NcepMnemonic {

  //| HEADR    | 362001 | TABLE D ENTRY - PROFILE COORDINATES                      |                      |
  private static final Pattern fields3 = Pattern.compile("^\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|\\s+(.*)\\s*\\|");
  private static final Pattern fields2 = Pattern.compile("^\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|");
  private static final Pattern fields5 = Pattern.compile("^\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|\\s+(.*)\\s+\\|");

  /**
   * Pattern to get 3 integers from beginning of line.
   */
  private static final Pattern ints6 = Pattern.compile("^\\d{6}");

  private static final int XlocalCutoff = 48;
  private static final int YlocalCutoff = 192;

  private static final boolean debugTable = false;

  /**
   * Read NCEP mnemonic BUFR tables.
   *
   * @return boolean
   * @throws IOException
   */
  public static boolean read(InputStream ios, BufrTables.Tables tables) throws IOException {
    if (ios == null)
      return false;

    if (tables.b == null)
      tables.b = new TableB("fake", "fake");
    if (tables.d == null)
      tables.d = new TableD("fake", "fake");

    HashMap<String, String> number = new HashMap<>(); // key = mnemonic value = fxy
    HashMap<String, String> desc = new HashMap<>(); // key = mnemonic value = description
    HashMap<String, String> mnseq = new HashMap<>();

    try {
      BufferedReader dataIS = new BufferedReader(new InputStreamReader(ios, CDM.utf8Charset));

      // read  mnemonic table
      Matcher m;
      // read header info and disregard
      while (true) {
        String line = dataIS.readLine();
        if (line == null) throw new RuntimeException("Bad NCEP mnemonic BUFR table ");
        if (line.contains("MNEMONIC")) break;
      }
      // read mnemonic, number, and description
      //| HEADR    | 362001 | TABLE D ENTRY - PROFILE COORDINATES                      |
      while (true) {
        String line = dataIS.readLine();
        if (line == null) break;
        if (line.contains("MNEMONIC")) break;
        if (line.contains("----")) continue;
        if (line.startsWith("*")) continue;
        if (line.startsWith("|       ")) continue;
        m = fields3.matcher(line);
        if (m.find()) {
          String mnu = m.group(1).trim();
          String fxy = m.group(2).trim();
          if (fxy.startsWith("3")) {
            number.put(mnu, fxy);
            desc.put(mnu, m.group(3).replace("TABLE D ENTRY - ", "").trim());
          } else if (fxy.startsWith("0")) {
            number.put(mnu, fxy);
            desc.put(mnu, m.group(3).replace("TABLE B ENTRY - ", "").trim());
          } else if (fxy.startsWith("A")) {
            number.put(mnu, fxy);
            desc.put(mnu, m.group(3).replace("TABLE A ENTRY - ", "").trim());
          }
        } else if (debugTable) {
          System.out.println("bad mnemonic, number, and description: " + line);
        }
      }
      // read in sequences using mnemonics
      //| ETACLS1  | HEADR {PROFILE} SURF FLUX HYDR D10M {SLYR} XTRA                   |
      while (true) {
        String line = dataIS.readLine();
        if (line == null) break;
        if (line.contains("MNEMONIC")) break;
        if (line.contains("----")) continue;
        if (line.startsWith("|       ")) continue;
        if (line.startsWith("*")) continue;
        m = fields2.matcher(line);
        if (m.find()) {
          String mnu = m.group(1).trim();
          if (mnseq.containsKey(mnu)) { // concat lines with same mnu
            String value = mnseq.get(mnu);
            value = value + " " + m.group(2);
            mnseq.put(mnu, value);
          } else {
            mnseq.put(mnu, m.group(2));
          }
        } else if (debugTable) {
          System.out.println("bad sequence mnemonic: " + line);
        }
      }

      // create sequences, replacing mnemonics with numbers
      for (Map.Entry<String, String> ent : mnseq.entrySet()) {
        String seq = ent.getValue();
        seq = seq.replaceAll("\\<", "1-1-0 0-31-0 ");
        seq = seq.replaceAll("\\>", "");
        seq = seq.replaceAll("\\{", "1-1-0 0-31-1 ");
        seq = seq.replaceAll("\\}", "");
        seq = seq.replaceAll("\\(", "1-1-0 0-31-2 ");
        seq = seq.replaceAll("\\)", "");

        StringTokenizer stoke = new StringTokenizer(seq, " ");
        List<Short> list = new ArrayList<>();
        while (stoke.hasMoreTokens()) {
          String mn = stoke.nextToken();
          if (mn.charAt(1) == '-') {
            list.add(Descriptor.getFxy(mn));
            continue;
          }
          // element descriptor needs hyphens
          m = ints6.matcher(mn);
          if (m.find()) {
            String F = mn.substring(0, 1);
            String X = removeLeading0(mn.substring(1, 3));
            String Y = removeLeading0(mn.substring(3));
            list.add(Descriptor.getFxy(F + "-" + X + "-" + Y));
            continue;
          }
          if (mn.startsWith("\"")) {
            int idx = mn.lastIndexOf('"');
            String count = mn.substring(idx + 1);
            list.add(Descriptor.getFxy("1-1-" + count));
            mn = mn.substring(1, idx);

          }
          if (mn.startsWith(".")) {
            String des = mn.substring(mn.length() - 4);
            mn = mn.replace(des, "....");

          }
          String fxy = number.get(mn);
          String F = fxy.substring(0, 1);
          String X = removeLeading0(fxy.substring(1, 3));
          String Y = removeLeading0(fxy.substring(3));
          list.add(Descriptor.getFxy(F + "-" + X + "-" + Y));
        }

        String fxy = number.get(ent.getKey());
        String X = removeLeading0(fxy.substring(1, 3));
        String Y = removeLeading0(fxy.substring(3));
        // these are in latest tables
        if (XlocalCutoff > Integer.parseInt(X) && YlocalCutoff > Integer.parseInt(Y))
          continue;
        //key = F + "-" + X + "-" + Y;

        short seqX = Short.parseShort(X.trim());
        short seqY = Short.parseShort(Y.trim());

        tables.d.addDescriptor(seqX, seqY, ent.getKey(), list);
        //short id = Descriptor.getFxy(key);
        //sequences.put(Short.valueOf(id), tableD);
      }

      // add some static repetition sequences
      // LOOK why?
      List<Short> list = new ArrayList<>();
      // 16 bit delayed repetition
      list.add(Descriptor.getFxy("1-1-0"));
      list.add(Descriptor.getFxy("0-31-2"));
      tables.d.addDescriptor((short) 60, (short) 1, "", list);
      //tableD = new DescriptorTableD("", "3-60-1", list, false);
      //tableD.put( "3-60-1", d);
      //short id = Descriptor.getFxy("3-60-1");
      //sequences.put(Short.valueOf(id), tableD);

      list = new ArrayList<>();
      // 8 bit delayed repetition
      list.add(Descriptor.getFxy("1-1-0"));
      list.add(Descriptor.getFxy("0-31-1"));
      tables.d.addDescriptor((short) 60, (short) 2, "", list);
      //tableD = new DescriptorTableD("", "3-60-2", list, false);
      //tableD.put( "3-60-2", d);
      //id = Descriptor.getFxy("3-60-2");
      //sequences.put(Short.valueOf(id), tableD);

      list = new ArrayList<>();
      // 8 bit delayed repetition
      list.add(Descriptor.getFxy("1-1-0"));
      list.add(Descriptor.getFxy("0-31-1"));
      tables.d.addDescriptor((short) 60, (short) 3, "", list);
      //tableD = new DescriptorTableD("", "3-60-3", list, false);
      //tableD.put( "3-60-3", d);
      //id = Descriptor.getFxy("3-60-3");
      //sequences.put(Short.valueOf(id), tableD);

      list = new ArrayList<>();
      // 1 bit delayed repetition
      list.add(Descriptor.getFxy("1-1-0"));
      list.add(Descriptor.getFxy("0-31-0"));
      tables.d.addDescriptor((short) 60, (short) 4, "", list);
      //tableD = new DescriptorTableD("", "3-60-4", list, false);
      //tableD.put( "3-60-4", d);
      //id = Descriptor.getFxy("3-60-4");
      //sequences.put(Short.valueOf(id), tableD);

      // add in element descriptors
      //  MNEMONIC | SCAL | REFERENCE   | BIT | UNITS
      //| FTIM     |    0 |           0 |  24 | SECONDS                  |-------------|

      //tableB = new TableB(tablename, tablename);

      while (true) {
        String line = dataIS.readLine();
        if (line == null) break;
        if (line.contains("MNEMONIC")) break;
        if (line.startsWith("|       ")) continue;
        if (line.startsWith("*")) continue;
        m = fields5.matcher(line);
        if (m.find()) {
          if (m.group(1).equals("")) {
            continue;
          } else if (number.containsKey(m.group(1).trim())) { // add descriptor to tableB
            String fxy = number.get(m.group(1).trim());
            String X = fxy.substring(1, 3);
            String Y = fxy.substring(3);
            String mnu = m.group(1).trim();
            String descr = desc.get(mnu);

            short x = Short.parseShort(X.trim());
            short y = Short.parseShort(Y.trim());

            // these are in latest tables so skip  LOOK WHY
            if (XlocalCutoff > x && YlocalCutoff > y)
              continue;

            int scale = Integer.parseInt(m.group(2).trim());
            int refVal = Integer.parseInt(m.group(3).trim());
            int width = Integer.parseInt(m.group(4).trim());
            String units = m.group(5).trim();

            tables.b.addDescriptor(x, y, scale, refVal, width, mnu, units, descr);

          } else if (debugTable) {
            System.out.println("bad element descriptors: " + line);
          }
        }
      }

    } finally {
      ios.close();
    }

    // LOOK why ?
    // default for NCEP
    // 0; 63; 0; 0; 0; 16; Numeric; Byte count
    tables.b.addDescriptor((short) 63, (short) 0, 0, 0, 16, "Byte count", "Numeric", null);

    return true;
  }

  private static String removeLeading0(String number) {
    if (number.length() == 2 && number.startsWith("0")) {
      number = number.substring(1);
    } else if (number.length() == 3 && number.startsWith("00")) {
      number = number.substring(2);
    } else if (number.length() == 3 && number.startsWith("0")) {
      number = number.substring(1);
    }
    return number;
  }

  /**
   * Read NCEP mnemonic BUFR tables.
   */
  private static void readSubCategories(String fileIn, PrintStream out, String token) throws IOException {
    System.out.printf("%s%n", fileIn);
    try (FileInputStream in = new  FileInputStream(fileIn)) {
      BufferedReader dataIS = new BufferedReader(new InputStreamReader(in,
              CDM.utf8Charset));
      while (true) {
        String line = dataIS.readLine();
        if (line == null) break;
        int posb = line.indexOf("DISCONTINUED");
        if (posb > 0) continue;
        posb = line.indexOf("NO LONGER");
        if (posb > 0) continue;
        posb = line.indexOf("WAS REPLACED");
        if (posb > 0) continue;

        int pos = line.indexOf(token);
        if (pos < 0) continue;
        System.out.printf("%s%n", line);

        boolean is31 = token.equals("031-");
        String subline = is31 ? line.substring(pos) : line.substring(pos + token.length());
        //if (is31) System.out.printf(" '%s'%n", subline);

        int pos2 = subline.indexOf(' ');
        String catS = subline.substring(0, pos2);
        String desc = subline.substring(pos2 + 1);
        //System.out.printf("   cat='%s'%n", catS);
        //System.out.printf("  desc='%s'%n", desc);
        int cat = Integer.parseInt(catS.substring(0, 3));
        int subcat = Integer.parseInt(catS.substring(4, 7));
        desc = StringUtil2.remove(desc, '|').trim();
        //System.out.printf("  cat=%d subcat=%d%n", cat,subcat);
        //System.out.printf("  desc='%s'%n", desc);
        //System.out.printf("%d, %d, %s%n", cat, subcat, desc);
        out.printf("%d; %d; %s%n", cat, subcat, desc);
      }
    }

  }

  public static void main(String args[]) throws IOException {
    String fileOut = "resource:/resources/bufrTables/local/ncep/DataSubCategories.csv";
    try (PrintStream pout = new PrintStream(fileOut, CDM.UTF8)) {
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.000.txt", pout, "MSG TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.001.txt", pout, "MESSAGE TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.002.txt", pout, "MSG TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.003.txt", pout, "MTYP ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.004.txt", pout, "MSG TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.005.txt", pout, "MSG TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.006.txt", pout, "M TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.007.txt", pout, "MTYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.008.txt", pout, "MSG TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.012.txt", pout, "M TYPE ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.021.txt", pout, "MTYP ");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.031.txt", pout, "031-");
      readSubCategories("C:/dev/github/thredds/bufr/src/main/sources/ncep/bufrtab.255.txt", pout, "MTYP ");
    }
    System.out.printf("=======================================%n");
    System.out.printf("%s%n", IO.readFile(fileOut));
    System.exit(0);

    String location = "resource:/resources/bufrTables/local/ncep/ncep.bufrtab.ETACLS1";
    try (InputStream ios = BufrTables.openStream(location)) {
      BufrTables.Tables tables = new BufrTables.Tables();
      NcepMnemonic.read(ios, tables);

      Formatter out = new Formatter(System.out);
      tables.b.show(out);
      tables.d.show(out);
    }

  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.httpservices;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.ByteArrayBody;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.message.BasicNameValuePair;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.apache.http.entity.ContentType.APPLICATION_OCTET_STREAM;
import static org.apache.http.entity.ContentType.TEXT_PLAIN;

/**
 * Class HTTPFormBuilder provides a mechanism for
 * constructing a postable entity for use with Post.
 * This code is derived from IDV HttpFormEntry.
 * The basic idea is that a list of HTTPFormBuilder.Field instances
 * is created to represent the fields and attachments of the form.
 * Then, a post method instance is provided and the content of the post
 * is set from the HTTPForm instance.
 */

public class HTTPFormBuilder
{
    //////////////////////////////////////////////////
    // Constants

    public static final Charset DFALTCHARSET = HTTPUtil.UTF8;

    //////////////////////////////////////////////////
    // Type Decls

    static protected enum Sort
    {
        TEXT, BYTES, STREAM, FILE;

        static public ContentType contenttype(Sort sort)
        {
            switch (sort) {
            case TEXT:
                return TEXT_PLAIN;
            case BYTES:
            case STREAM:
            case FILE:
            default:
                return APPLICATION_OCTET_STREAM;
            }
        }

        static public String mimetype(Sort sort)
        {
            return contenttype(sort).getMimeType();
        }
    }

    static protected class Field
    {
        public Sort sort;
        public String fieldname = null;
        public Object value = null;
        public String name = null; // typically file name

        public Field(Sort sort, String fieldname, Object value, String name)
        {
            this.sort = sort;
            this.fieldname = fieldname;
            this.value = value;
            this.name = name;
        }
    }

    //////////////////////////////////////////////////
    // Instance variables

    protected Map<String, Field> parts = new HashMap<>();

    protected Charset charset = DFALTCHARSET;

    protected boolean usemultipart = false;

    //////////////////////////////////////////////////
    // Constructor(s)

    public HTTPFormBuilder()
    {
    }

    //////////////////////////////////////////////////
    // Accessors

    public String getCharset()
    {
        return this.charset.displayName();
    }

    public HTTPFormBuilder setCharset(String charset)
    {
        this.charset = Charset.forName(charset);
	return this;
    }

    public HTTPFormBuilder setCharset(Charset charset)
    {
        this.charset = charset;
	return this;
    }
    //////////////////////////////////////////////////
    // Field construction

    public HTTPFormBuilder
    add(String fieldname, String text)
            throws HTTPException
    {
        if(fieldname == null || text == null || fieldname.length() == 0)
            throw new IllegalArgumentException();
        Field f = new Field(Sort.TEXT, fieldname, text, null);
        parts.put(fieldname, f);
	return this;
    }

    public HTTPFormBuilder
    add(String fieldname, byte[] content, String filename)
            throws HTTPException
    {
        if(isempty(fieldname))
            throw new IllegalArgumentException();
        if(content == null) content = new byte[0];
        if(isempty(filename)) filename = "";

        Field f = new Field(Sort.BYTES, fieldname, content, filename);
        parts.put(fieldname, f);
        this.usemultipart = true;
	return this;
    }

    public HTTPFormBuilder
    add(String fieldname, final InputStream content, String filename)
            throws HTTPException
    {
        if(isempty(fieldname) || content == null || isempty(filename))
            throw new IllegalArgumentException();
        Field f = new Field(Sort.STREAM, fieldname, content, filename);
        parts.put(fieldname, f);
        this.usemultipart = true;
	return this;
    }

    public HTTPFormBuilder
    add(String fieldname, File content)
            throws HTTPException
    {
        if(isempty(fieldname) || content == null)
            throw new IllegalArgumentException();
        Field f = new Field(Sort.FILE, fieldname, content, content.getName());
        parts.put(fieldname, f);
        this.usemultipart = true;
	return this;
    }

    public HttpEntity build()
            throws HTTPException
    {
        if(this.usemultipart)
            return buildmultipart();
        else
            return buildsimple();
    }

    protected HttpEntity buildsimple()
            throws HTTPException
    {
        List<NameValuePair> params = new ArrayList<>();
        for(Map.Entry<String, Field> mapentry : parts.entrySet()) {
            Field field = mapentry.getValue();
            params.add(new BasicNameValuePair(field.fieldname, field.value.toString()));
        }
        try {
            return new UrlEncodedFormEntity(params);
        } catch (UnsupportedEncodingException e) {
            return null;
        }
    }

    protected HttpEntity buildmultipart()
            throws HTTPException
    {
        MultipartEntityBuilder mpb = MultipartEntityBuilder.create();
        mpb.setCharset(this.charset);
        mpb.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
        for(Map.Entry<String, Field> mapentry : parts.entrySet()) {
            Field field = mapentry.getValue();
            Sort sort = field.sort;
            ContentType ct = Sort.contenttype(sort);
            ContentBody body = null;
            switch (sort) {
            case TEXT:
                body = new StringBody(field.value.toString(), ct);
                break;
            case BYTES:
                body = new ByteArrayBody((byte[]) field.value, field.name);
                break;
            case STREAM:
                // There appears to be a bug that make direct use of inputstream
                // fail with a 411 http code.
                // Temporary workaround is to read the input stream
                // as a set of bytes
                try {
                    byte[] tmp = HTTPUtil.readbinaryfile((InputStream) field.value);
                    body = new ByteArrayBody(tmp, field.name);
                } catch (IOException ioe) {
                    throw new HTTPException(ioe);
                }
                break;
            case FILE:
                body = new FileBody((File) field.value, ct, field.name);
                break;
            }
            mpb.addPart(field.fieldname, body);
        }
        return mpb.build();
    }

    protected boolean isempty(String x)
    {
        return x == null || x.length() == 0;
    }
}


/*
 * Copyright (c) 1998 - 2014. University Corporation for Atmospheric Research/Unidata
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package thredds.featurecollection;

import org.jdom2.Element;
import org.jdom2.input.SAXBuilder;
import thredds.client.catalog.Catalog;
import thredds.inventory.CollectionAbstract;
import ucar.nc2.util.AliasTranslator;

import java.io.File;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;

/**
 * Read the <featureCollection> element. create a FeatureCollectionConfig
 *
 * @author John
 * @since 1/17/2015
 */
public class FeatureCollectionConfigBuilder {
  Formatter errlog;
  public boolean fatalError;

  public FeatureCollectionConfigBuilder(Formatter errlog) {
    this.errlog = errlog;
  }

  // input is xml file with just the <featureCollection>
  public FeatureCollectionConfig readConfigFromFile(String filename) {

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      doc = builder.build(filename);
    } catch (Exception e) {
      System.out.printf("Error parsing featureCollection %s err = %s", filename, e.getMessage());
      return null;
    }

    return readConfig(doc.getRootElement());
  }

  /**
   * Read a catalog and extract a FeatureCollectionConfig from it
   *
   * @param catalogAndPath catalog filename, or catalog#featureName
   * @return FeatureCollectionConfig or null
   */
  public FeatureCollectionConfig readConfigFromCatalog(String catalogAndPath) {
    String catFilename;
    String fcName = null;

    int pos = catalogAndPath.indexOf("#");
    if (pos > 0) {
      catFilename = catalogAndPath.substring(0, pos);
      fcName = catalogAndPath.substring(pos + 1);
    } else {
      catFilename = catalogAndPath;
    }

    File cat = new File(catFilename);
    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      doc = builder.build(cat);
    } catch (Exception e) {
      e.printStackTrace();
      return null;
    }

    try {
      List<Element> fcElems = new ArrayList<>();
      findFeatureCollection(doc.getRootElement(), fcName, fcElems);
      if (fcElems.size() > 0)
        return readConfig(fcElems.get(0));

    } catch (IllegalStateException e) {
      e.printStackTrace();
    }

    return null;
  }

  private void findFeatureCollection(Element parent, String name, List<Element> fcElems) {
    List<Element> elist = parent.getChildren("featureCollection", Catalog.defNS);
    if (name == null)
      fcElems.addAll(elist);
    else {
      for (Element elem : elist) {
        if (name.equals(elem.getAttributeValue("name")))
          fcElems.add(elem);
      }
    }
    for (Element child : parent.getChildren("dataset", Catalog.defNS))
      findFeatureCollection(child, name, fcElems);
  }

  public FeatureCollectionConfig readConfig(Element featureCollectionElement) {
    String name = featureCollectionElement.getAttributeValue("name");
    String path = featureCollectionElement.getAttributeValue("path");
    String fcTypeS = featureCollectionElement.getAttributeValue("featureType");

    FeatureCollectionType fcType = FeatureCollectionType.valueOf(fcTypeS);
    if (fcType == null) {
      errlog.format("featureCollection %s must have a valid FeatureCollectionType attribute, found '%s'%n", name, fcTypeS);
      fatalError = true;
    }

    // collection element required
    Element collElem = featureCollectionElement.getChild("collection", Catalog.defNS);
    if (collElem == null) {
      errlog.format("featureCollection %s must have a <collection> element%n", name);
      fatalError = true;
      return null;
    }
    String collectionName = collElem.getAttributeValue("name");
    collectionName = CollectionAbstract.cleanName(collectionName != null ? collectionName : name);

    String spec = collElem.getAttributeValue("spec");
      spec = expandAliasForCollectionSpec(spec);
    String timePartition = collElem.getAttributeValue("timePartition");
    String dateFormatMark = collElem.getAttributeValue("dateFormatMark");
    String olderThan = collElem.getAttributeValue("olderThan");
    String rootDir = collElem.getAttributeValue("rootDir");
    String regExp = collElem.getAttributeValue("regExp");
    if (spec == null && rootDir == null) {
      errlog.format("featureCollection " + name + " must have a spec or rootDir attribute.");
      fatalError = true;
      return null;
    }
    Element innerNcml = featureCollectionElement.getChild("netcdf", Catalog.ncmlNS);
    FeatureCollectionConfig config = new FeatureCollectionConfig(name, path, fcType, spec, collectionName, dateFormatMark, olderThan,
            timePartition, innerNcml);
    config.setFilter(rootDir, regExp);
    config.setFilesSort( featureCollectionElement.getChild("filesSort", Catalog.defNS));

    // tds and update elements
    Element tdmElem = featureCollectionElement.getChild("tdm", Catalog.defNS);
    config.tdmConfig = readUpdateElement(tdmElem);
    Element updateElem = featureCollectionElement.getChild("update", Catalog.defNS);
    config.updateConfig = readUpdateElement(updateElem);

    // protoDataset element
    Element protoElem = featureCollectionElement.getChild("protoDataset", Catalog.defNS);
    if (protoElem != null) {
      String choice = protoElem.getAttributeValue("choice");
      String change = protoElem.getAttributeValue("change");
      String param = protoElem.getAttributeValue("param");
      Element ncmlElem = protoElem.getChild("netcdf", Catalog.ncmlNS);
      config.protoConfig = new FeatureCollectionConfig.ProtoConfig(choice, change, param, ncmlElem);
    }

    // fmrcConfig element
    Element fmrcElem = featureCollectionElement.getChild("fmrcConfig", Catalog.defNS);
    if (fmrcElem != null) {
      String regularize = fmrcElem.getAttributeValue("regularize");
      config.fmrcConfig = new FeatureCollectionConfig.FmrcConfig(regularize);

      String datasetTypes = fmrcElem.getAttributeValue("datasetTypes");
      if (null != datasetTypes)
        config.fmrcConfig.addDatasetType(datasetTypes);

      List<Element> bestElemsOld = fmrcElem.getChildren("dataset", Catalog.defNS);
      for (Element best : bestElemsOld) {
        String bestName = best.getAttributeValue("name");
        String offs = best.getAttributeValue("offsetsGreaterEqual");
        double off = Double.parseDouble(offs);
        config.fmrcConfig.addBestDataset(bestName, off);
      }
      List<Element> bestElems = fmrcElem.getChildren("bestDataset", Catalog.defNS);
      for (Element best : bestElems) {
        String bestName = best.getAttributeValue("name");
        String offs = best.getAttributeValue("offsetsGreaterEqual");
        double off = Double.parseDouble(offs);
        config.fmrcConfig.addBestDataset(bestName, off);
      }
    }

    // pointConfig element optional
    Element pointElem = featureCollectionElement.getChild("pointConfig", Catalog.defNS);
    if (pointElem != null) {
      String datasetTypes = pointElem.getAttributeValue("datasetTypes");
      if (null != datasetTypes)
        config.pointConfig.addDatasetType(datasetTypes);
    }

    // gribConfig element optional
    Element gribConfig = featureCollectionElement.getChild("gribConfig", Catalog.defNS);
    if (gribConfig != null) {
      config.gribConfig.configFromXml(gribConfig, Catalog.defNS);
    }

    // done reading - do anything needed
    config.finish();

    return config;
  }

  private FeatureCollectionConfig.UpdateConfig readUpdateElement(Element updateElem) {
    if (updateElem == null) {
      return new FeatureCollectionConfig.UpdateConfig(); // default
    }

    String startup = updateElem.getAttributeValue("startup");
    String rewrite = updateElem.getAttributeValue("rewrite");
    String recheckAfter = updateElem.getAttributeValue("recheckAfter");
    String rescan = updateElem.getAttributeValue("rescan");
    String trigger = updateElem.getAttributeValue("trigger");
    String deleteAfter = updateElem.getAttributeValue("deleteAfter");

    return new FeatureCollectionConfig.UpdateConfig(startup, rewrite, recheckAfter, rescan, trigger, deleteAfter);
  }

  private String expandAliasForCollectionSpec(String location) {
    String result = AliasTranslator.translateAlias(location);
    if (result != null) return result;
    return location;
  }

}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.core.dmr;

import dap4.core.util.DapException;
import dap4.core.util.DapSort;

import java.util.ArrayList;
import java.util.List;

/**
 * This class defines a non-groupd group:
 * i.e. one with an atomic type.
 */

public class DapGroup extends DapNode implements DapDecl
{

    //////////////////////////////////////////////////
    // Instance variables

    protected List<DapNode> decls = new ArrayList<DapNode>();
    protected List<DapGroup> groups = new ArrayList<DapGroup>();
    protected List<DapEnumeration> enums = new ArrayList<DapEnumeration>();
    protected List<DapDimension> dimensions = new ArrayList<DapDimension>();
    protected List<DapVariable> variables = new ArrayList<DapVariable>();
    protected List<DapStructure> compounds = new ArrayList<DapStructure>();

    //////////////////////////////////////////////////
    // Constructors

    public DapGroup()
    {
        super();
    }

    public DapGroup(String name)
    {
        super(name);
    }

    //////////////////////////////////////////////////
    // Accessors

    public List<DapNode> getDecls()
    {
        return decls;
    }

    public void
    setDecls(List<? extends DapNode> decls)
            throws DapException
    {
        decls.clear();
        groups.clear();
        enums.clear();
        dimensions.clear();
        variables.clear();
        compounds.clear();
        for(DapNode node : decls) {
            addDecl(node);
        }
    }

    /**
     * Add single declaration
     */

    public void
    addDecl(DapNode newdecl)
            throws DapException
    {
        DapSort newsort = newdecl.getSort();
        String newname = newdecl.getShortName();
        boolean suppress = false;
        // Look for name conflicts (ignore anonymous dimensions)
        if(newsort != DapSort.DIMENSION || newname != null) {
            for(DapNode decl : decls) {
                if(newsort == decl.getSort()
                        && newname.equals(decl.getShortName()))
                    throw new DapException("DapGroup: attempt to add duplicate decl: " + newname);
            }
        } else { // Anonymous
            DapDimension anon = (DapDimension) newdecl;
            assert (newsort == DapSort.DIMENSION && newname == null);
            // Search for matching anonymous dimension
            boolean found = false;
            for(DapDimension dim : dimensions) {
                if(!dim.isShared() && dim.getSize() == anon.getSize()) {
                    found = true;
                    break;
                }
            }
            // Define the anondecl in root group
            if(!found && !isTopLevel()) getDataset().addDecl(anon);
            suppress = found || !isTopLevel();
        }
        if(!suppress) {
            decls.add(newdecl);
            newdecl.setParent(this); // Cross link
        }
        switch (newdecl.getSort()) {
        case ATTRIBUTE:
        case ATTRIBUTESET:
        case OTHERXML:
            super.addAttribute((DapAttribute) newdecl);
            break;
        case DIMENSION:
            if(!suppress)
                dimensions.add((DapDimension) newdecl);
            break;
        case ENUMERATION:
            enums.add((DapEnumeration) newdecl);
            break;
        case ATOMICTYPE:
            break; // do nothing
        case STRUCTURE:
        case SEQUENCE:
            compounds.add((DapStructure)newdecl);
            break;
        case VARIABLE:
            variables.add((DapVariable) newdecl);
            break;
        case GROUP:
        case DATASET:
            if(this != (DapGroup) newdecl)
                groups.add((DapGroup) newdecl);
            break;
        default:
            throw new ClassCastException(newdecl.getShortName());
        }
    }

    /**
     * We will need to re-order the groups
     */

    void
    updateGroups(List<DapGroup> groups)
    {
        // Verify that the incoming groups are all and only in the list of groups.
        assert (groups.size() == this.groups.size()) : "Update groups: not same size";
        for(DapGroup g : groups) {
            if(!this.groups.contains(g))
                assert (false) : "Update groups: attempt to add new group";
        }
    }

    public List<DapGroup> getGroups()
    {
        return groups;
    }

    public List<DapEnumeration> getEnums()
    {
        return enums;
    }

    public List<DapStructure> getCompounds()
        {
            return compounds;
        }

    public List<DapDimension> getDimensions()
    {
        return dimensions;
    }

    public List<DapVariable> getVariables()
    {
        return variables;
    }

    //////////////////////////////////////////////////
    // Lookup Functions

    public DapNode
    findByName(String name, DapSort... sortset)
    {
        return findInGroup(name, sortset);
    }

    public DapNode
    findInGroup(String name, DapSort... sortset)
    {
        for(DapSort sort : sortset) {
            switch (sort) {
            case ATTRIBUTE:
            case ATTRIBUTESET:
            case OTHERXML:
                DapAttribute attr = super.getAttributes().get(name);
                return attr;
            case DIMENSION:
                for(DapDimension x : dimensions) {
                    if(x.getShortName() != null && x.getShortName().equals(name))
                        return x;
                }
                break;
            case ENUMERATION:
                for(DapEnumeration x : enums) {
                    if(x.getShortName().equals(name))
                        return x;
                }
                break;
            case STRUCTURE:
                for(DapStructure d : compounds) {
                    if(d.getSort() == DapSort.STRUCTURE) {
                        if(d.getShortName().equals(name))
                            return d;
                    }
                }
                break;
            case SEQUENCE:
                for(DapStructure d : compounds) {
                    if(d.getSort() == DapSort.SEQUENCE) {
                        if(d.getShortName().equals(name))
                            return d;
                    }
                }
                break;
            case VARIABLE:
                for(DapVariable x : variables) {
                    if(x.getShortName().equals(name))
                        return x;
                }
                break;
            case GROUP:
                for(DapGroup x : groups) {
                    if(x.getShortName().equals(name))
                        return (x);
                }
                break;
            default:
                break;
            }
        }
        return null;
    }

    /**
     * Parse an FQN and use it to trace to a specific
     * object in a dataset. Absolute FQN paths are passed
     * to DapDataset.findByFQN(). Relative FQNs are assumed
     * to be WRT to the FQN of this node
     *
     * @param fqn     the fully qualified name
     * @param sortset the kinds of object we are looking for
     * @return the matching Dap Node or null if not found
     */

    public DapNode
    findByFQN(String fqn, DapSort... sortset)
            throws DapException
    {
        fqn = fqn.trim();
        if(fqn == null)
            return null;
        if(fqn.charAt(0) != '/') {
            String prefix = this.getFQN();
            fqn = prefix + '/' + fqn;
        }
        return getDataset().lookup(fqn, sortset);
    }

    /**
     * Locate a variable in this group
     *
     * @param name the variable's name
     * @return the matching Dap Node or null if not found
     */
    public DapVariable
    findVariable(String name)
    {
        DapNode var = findInGroup(name, DapSort.VARIABLE);
        return (DapVariable) var;
    }

}// Class DapGroup


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

// $Id: TestCompare.java 51 2006-07-12 17:13:13Z caron $

package ucar.nc2.util;

import ucar.nc2.constants.CDM;
import ucar.nc2.dataset.*;
import ucar.nc2.*;
import ucar.ma2.*;
import ucar.nc2.iosp.netcdf4.Nc4;

import java.io.*;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Formatter;
import java.util.ArrayList;

/**
 * Compare two NetcdfFile.
 * Dont use assert, place results in Formatter.
 *
 * @author john
 */
public class CompareNetcdf2 {

  public interface ObjFilter {
    // if true, compare attribute, else skip comparision
    boolean attCheckOk(Variable v, Attribute att);

    // if true, compare variable, else skip comparision
    boolean varDataTypeCheckOk(Variable v);
  }

  public static class Netcdf4ObjectFilter implements ObjFilter {
    @Override
    public boolean attCheckOk(Variable v, Attribute att) {
      // if (v != null && v.isMemberOfStructure()) return false;
      String name = att.getShortName();

      // added by cdm
      if (name.equals(CDM.CHUNK_SIZES)) return false;
      if (name.equals(CDM.FILL_VALUE)) return false;
      if (name.equals("_lastModified")) return false;

      // hidden by nc4
      if (name.equals(Nc4.NETCDF4_DIMID)) return false;  // preserve the order of the dimensions
      if (name.equals(Nc4.NETCDF4_COORDINATES)) return false;  // ??
      if (name.equals(Nc4.NETCDF4_STRICT)) return false;

      if(name.startsWith("_")) return false;

      // not implemented yet
      //if (att.getDataType().isEnum()) return false;

      return true;
    }

    @Override
    public boolean varDataTypeCheckOk(Variable v) {
      if (v.getDataType() == DataType.CHAR) return false;    // temp workaround
      return v.getDataType() != DataType.STRING;
    }
  }

  static public boolean compareFiles(NetcdfFile org, NetcdfFile copy, Formatter f) {
    return compareFiles(org, copy, f, false, false, false);
  }

  static public boolean compareFiles(NetcdfFile org, NetcdfFile copy, Formatter f, boolean _compareData, boolean _showCompare, boolean _showEach) {
    CompareNetcdf2 tc = new CompareNetcdf2(f, _showCompare, _showEach, _compareData);
    return tc.compare(org, copy);
  }

  static public boolean compareLists(List org, List copy, Formatter f) {
    boolean ok1 = checkContains("first", org, copy, f);
    boolean ok2 = checkContains("second", copy, org, f);
    return ok1 && ok2;
  }

  static private boolean checkContains(String what, List container, List wantList, Formatter f) {
    boolean ok = true;

    for (Object want1 : wantList) {
      int index2 = container.indexOf(want1);
      if (index2 < 0) {
        f.format("  ** %s missing in %s %n", want1, what);
        ok = false;
      }
    }

    return ok;
  }

  /////////

  private Formatter f;
  private boolean showCompare = false;
  private boolean showEach = false;
  private boolean compareData = false;

  public CompareNetcdf2() {
    this(new Formatter(System.out));
  }

  public CompareNetcdf2(Formatter f) {
    this(f, false, false, false);
  }

  public CompareNetcdf2(Formatter f, boolean showCompare, boolean showEach, boolean compareData) {
    this.f = f;
    this.compareData = compareData;
    this.showCompare = showCompare;
    this.showEach = showEach;
  }

  public boolean compare(NetcdfFile org, NetcdfFile copy) {
    return compare(org, copy, showCompare, showEach, compareData);
  }

  public boolean compare(NetcdfFile org, NetcdfFile copy, boolean showCompare, boolean showEach, boolean compareData) {
    return compare(org, copy, null, showCompare, showEach, compareData);
  }

  public boolean compare(NetcdfFile org, NetcdfFile copy, ObjFilter filter, boolean showCompare, boolean showEach, boolean compareData) {
    this.compareData = compareData;
    this.showCompare = showCompare;
    this.showEach = showEach;

    f.format(" First file = %s%n", org.getLocation());
    f.format(" Second file= %s%n", copy.getLocation());

    long start = System.currentTimeMillis();

    boolean ok = compareGroups(org.getRootGroup(), copy.getRootGroup(), filter);
    f.format(" Files are the same = %s%n", ok);

    long took = System.currentTimeMillis() - start;
    f.format(" Time to compare = %d msecs%n", took);

    // coordinate systems
    if (org instanceof NetcdfDataset && copy instanceof NetcdfDataset) {
      NetcdfDataset orgds = (NetcdfDataset) org;
      NetcdfDataset copyds = (NetcdfDataset) copy;

      List matches = new ArrayList();
      ok &= checkAll("Dataset CS:", orgds.getCoordinateSystems(), copyds.getCoordinateSystems(), matches);
      for (int i = 0; i < matches.size(); i += 2) {
        CoordinateSystem orgCs = (CoordinateSystem) matches.get(i);
        CoordinateSystem copyCs = (CoordinateSystem) matches.get(i + 1);
        ok &= compareCoordinateSystem(orgCs, copyCs, filter);
      }
    }

    return ok;
  }

  public boolean compareVariables(NetcdfFile org, NetcdfFile copy) {
    f.format("Original = %s%n", org.getLocation());
    f.format("CompareTo= %s%n", copy.getLocation());
    boolean ok = true;

    for (Variable orgV : org.getVariables()) {
      //if (orgV.isCoordinateVariable()) continue;

      Variable copyVar = copy.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" MISSING '%s' in 2nd file%n", orgV.getFullName());
        ok = false;
      } else {
        ok &= compareVariables(orgV, copyVar, null, compareData, true);
      }
    }

    f.format("%n");
    for (Variable orgV : copy.getVariables()) {
      //if (orgV.isCoordinateVariable()) continue;
      Variable copyVar = org.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" MISSING '%s' in 1st file%n", orgV.getFullName());
        ok = false;
      }
    }

    return ok;
  }

  /* private boolean compare(List<Dimension> dims1, List<Dimension> dims2) {
    if (dims1.size() != dims2.size()) return false;
    for (int i = 0; i < dims1.size(); i++) {
      Dimension dim1 = dims1.get(i);
      Dimension dim2 = dims2.get(i);
      //if (!dim1.getName().equals(dim2.getName())) return false;
      if (dim1.getLength() != dim2.getLength()) return false;
    }
    return true;
  }  */

  private boolean compareGroups(Group org, Group copy, ObjFilter filter) {
    if (showCompare) f.format("compare Group %s to %s %n", org.getShortName(), copy.getShortName());
    boolean ok = true;

    if (!org.getShortName().equals(copy.getShortName())) {
      f.format(" ** names are different %s != %s %n", org.getShortName(), copy.getShortName());
      ok = false;
    }

    // dimensions
    ok &= checkDimensions(org.getDimensions(), copy.getDimensions());
    ok &= checkDimensions(copy.getDimensions(), org.getDimensions());

    // attributes
    ok &= checkAttributes(null, org.getAttributes(), copy.getAttributes(), filter);

    // enums
    ok &= checkEnums(org, copy);

    // variables
    // cant use object equality, just match on short name
    for (Variable orgV : org.getVariables()) {
      Variable copyVar = copy.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" ** cant find variable %s in 2nd file%n", orgV.getFullName());
        ok = false;
      } else {
        ok &= compareVariables(orgV, copyVar, filter, compareData, true);
      }
    }

    for (Variable copyV : copy.getVariables()) {
      Variable orgV = org.findVariable(copyV.getShortName());
      if (orgV == null) {
        f.format(" ** cant find variable %s in 1st file%n", copyV.getFullName());
        ok = false;
      }
    }

    // nested groups
    List groups = new ArrayList();
    String name = org.isRoot() ? "root" : org.getFullName();
    ok &= checkAll(name, org.getGroups(), copy.getGroups(), groups);
    for (int i = 0; i < groups.size(); i += 2) {
      Group orgGroup = (Group) groups.get(i);
      Group copyGroup = (Group) groups.get(i + 1);
      ok &= compareGroups(orgGroup, copyGroup, filter);
    }

    return ok;
  }

  public boolean compareVariable(Variable org, Variable copy) {
    return compareVariables(org, copy, null, compareData, true);
  }

  private boolean compareVariables(Variable org, Variable copy, ObjFilter filter, boolean compareData, boolean justOne) {
    boolean ok = true;

    if (showCompare) f.format("compare Variable %s to %s %n", org.getFullName(), copy.getFullName());
    if (!org.getFullName().equals(copy.getFullName())) {
      f.format(" ** names are different %s != %s %n", org.getFullName(), copy.getFullName());
      ok = false;
    }
    if (filter != null && filter.varDataTypeCheckOk(org) && (org.getDataType() != copy.getDataType())) {
      f.format(" ** %s dataTypes are different %s != %s %n", org.getFullName(), org.getDataType(), copy.getDataType());
      ok = false;
    }

    // dimensions
    ok &= checkDimensions(org.getDimensions(), copy.getDimensions());
    ok &= checkDimensions(copy.getDimensions(), org.getDimensions());

    // attributes
    ok &= checkAttributes(org, org.getAttributes(), copy.getAttributes(), filter);

    // data !!
    if (compareData) {
      try {
        ok &= compareVariableData(org, copy, showCompare, justOne);

      } catch (IOException e) {
        StringWriter sw = new StringWriter(5000);
        e.printStackTrace(new PrintWriter(sw));
        f.format("%s", sw.toString());
        return false;
      }
    }

    // nested variables
    if (org instanceof Structure) {
      if (!(copy instanceof Structure)) {
        f.format("  ** %s not Structure%n", org);
        ok = false;

      } else {
        Structure orgS = (Structure) org;
        Structure ncmlS = (Structure) copy;

        List vars = new ArrayList();
        ok &= checkAll("struct " + orgS.getNameAndDimensions(), orgS.getVariables(), ncmlS.getVariables(), vars);
        for (int i = 0; i < vars.size(); i += 2) {
          Variable orgV = (Variable) vars.get(i);
          Variable ncmlV = (Variable) vars.get(i + 1);
          ok &= compareVariables(orgV, ncmlV, filter, false, true);
        }
      }
    }

    // coordinate systems
    if (org instanceof VariableEnhanced && copy instanceof VariableEnhanced) {
      VariableEnhanced orgds = (VariableEnhanced) org;
      VariableEnhanced copyds = (VariableEnhanced) copy;

      List matches = new ArrayList();
      ok &= checkAll(orgds.getFullName(), orgds.getCoordinateSystems(), copyds.getCoordinateSystems(), matches);
      for (int i = 0; i < matches.size(); i += 2) {
        CoordinateSystem orgCs = (CoordinateSystem) matches.get(i);
        CoordinateSystem copyCs = (CoordinateSystem) matches.get(i + 1);
        ok &= compareCoordinateSystem(orgCs, copyCs, filter);
      }
    }

    return ok;
  }

  private boolean compareCoordinateSystem(CoordinateSystem cs1, CoordinateSystem cs2, ObjFilter filter) {
    if (showCompare)
      f.format("compare CoordinateSystem '%s' to '%s' %n", cs1.getName(), cs2.getName());

    List matchAxes = new ArrayList();
    boolean ok = checkAll(cs1.getName(), cs1.getCoordinateAxes(), cs2.getCoordinateAxes(), matchAxes);
    for (int i = 0; i < matchAxes.size(); i += 2) {
      CoordinateAxis orgCs = (CoordinateAxis) matchAxes.get(i);
      CoordinateAxis copyCs = (CoordinateAxis) matchAxes.get(i + 1);
      ok &= compareCoordinateAxis(orgCs, copyCs, filter);
    }

    List matchTransforms = new ArrayList();
    ok &= checkAll(cs1.getName(), cs1.getCoordinateTransforms(), cs2.getCoordinateTransforms(), matchTransforms);
    return ok;
  }

  private boolean compareCoordinateAxis(CoordinateAxis a1, CoordinateAxis a2, ObjFilter filter) {
    if (showCompare)
      f.format("  compare CoordinateAxis '%s' to '%s' %n", a1.getShortName(), a2.getShortName());

    compareVariable(a1, a2);
    return true;
  }

  // make sure each object in wantList is contained in container, using equals().

  // make sure each object in each list are in the other list, using equals().
  // return an arrayList of paired objects.

  private boolean checkAttributes(Variable v, List<Attribute> list1, List<Attribute> list2, ObjFilter filter) {
    boolean ok = true;

    String name = v == null ? "global" : "variable " + v.getFullName();
    for (Attribute att1 : list1) {
      if (filter == null || filter.attCheckOk(v, att1))
        ok &= checkEach(name, att1, "file1", list1, "file2", list2, null);
    }

    for (Attribute att2 : list2) {
      if (filter == null || filter.attCheckOk(v, att2))
        ok &= checkEach(name, att2, "file2", list2, "file1", list1, null);
    }

    return ok;
  }

  private boolean checkDimensions(List<Dimension> list1, List<Dimension> list2) {
    boolean ok = true;

    for (Dimension d1 : list1) {
      if (d1.isShared()) {
        boolean hasit = list2.contains(d1);
        if (!hasit)
          f.format("  ** Missing dim %s not in file2 %n", d1);
        ok &= hasit;
      }
    }

    return ok;
  }

  // make sure each object in each list are in the other list, using equals().
  // return an arrayList of paired objects.

  private boolean checkEnums(Group org, Group copy) {
    boolean ok = true;

    for (EnumTypedef enum1 : org.getEnumTypedefs()) {
      if (showCompare) f.format("compare Enum %s%n", enum1.getShortName());
      EnumTypedef enum2 = copy.findEnumeration(enum1.getShortName());
      if (enum2 == null) {
        f.format("  ** Enum %s not in file2 %n", enum1.getShortName());
        ok = false;
        continue;
      }
      if (!enum1.equals(enum2)) {
        f.format("  ** Enum %s not equal%n  %s%n  %s%n", enum1.getShortName(), enum1, enum2);
        ok = false;
      }
    }

    for (EnumTypedef enum2 : copy.getEnumTypedefs()) {
      EnumTypedef enum1 = org.findEnumeration(enum2.getShortName());
      if (enum1 == null) {
        f.format("  ** Enum %s not in file1 %n", enum2.getShortName());
        ok = false;
      }
    }
    return ok;
  }

  // make sure each object in each list are in the other list, using equals().
  // return an arrayList of paired objects.

  private boolean checkAll(String what, List list1, List list2, List result) {
    boolean ok = true;

    for (Object aList1 : list1) {
      ok &= checkEach(what, aList1, "file1", list1, "file2", list2, result);
    }

    for (Object aList2 : list2) {
      ok &= checkEach(what, aList2, "file2", list2, "file1", list1, null);
    }

    return ok;
  }

  // check that want is in both list1 and list2, using object.equals()

  private boolean checkEach(String what, Object want1, String name1, List list1, String name2, List list2, List result) {
    boolean ok = true;
    try {
      int index2 = list2.indexOf(want1);
      if (index2 < 0) {
        f.format("  ** %s: %s 0x%x (%s) not in %s %n", what, want1, want1.hashCode(), name1, name2);
        ok = false;
      } else { // found it in second list
        Object want2 = list2.get(index2);
        int index1 = list1.indexOf(want2);
        if (index1 < 0) { // can this happen ??
          f.format("  ** %s: %s 0x%x (%s) not in %s %n", what, want2, want2.hashCode(), name2, name1);
          ok = false;

        } else { // found it in both lists
          Object want = list1.get(index1);
          if (!want.equals(want1)) {
            f.format("  ** %s: %s 0x%x (%s) not equal to %s 0x%x (%s) %n", what, want1, want1.hashCode(), name1, want2, want2.hashCode(), name2);
            ok = false;
          } else {
            if (showEach)
              f.format("  OK <%s> equals <%s>%n", want1, want2);
            if (result != null) {
              result.add(want1);
              result.add(want2);
            }
          }
        }
      }

    } catch (Throwable t) {
      t.printStackTrace();
      f.format(" *** Throwable= %s %n", t.getMessage());
    }

    return ok;
  }

  private boolean compareVariableData(Variable var1, Variable var2, boolean showCompare, boolean justOne) throws IOException {
    Array data1 = var1.read();
    Array data2 = var2.read();

    if (showCompare)
      f.format(" compareArrays %s unlimited=%s size=%d%n", var1.getNameAndDimensions(), var1.isUnlimited(), data1.getSize());
    boolean ok = compareData(var1.getFullName(), data1, data2, justOne);
    if (showCompare) f.format("   ok=%s%n", ok);
    return ok;
  }

  public boolean compareData(String name, Array data1, double[] data2) {
    Array data2a = Array.factory(DataType.DOUBLE, new int[]{data2.length}, data2);
    return compareData(name, data1, data2a, false, false);
  }

  public boolean compareData(String name, double[] data1, double[] data2) {
    Array data1a = Array.factory(DataType.DOUBLE, new int[]{data1.length}, data1);
    Array data2a = Array.factory(DataType.DOUBLE, new int[]{data2.length}, data2);
    return compareData(name, data1a, data2a, false, false);
  }

  public boolean compareData(String name, Array data1, Array data2, boolean justOne) {
    return compareData(name, data1, data2, justOne, true);
  }

  public boolean compareData(String name, Array data1, Array data2) {
    return compareData(name, data1, data2, false, true);
  }

  private boolean compareData(String name, Array data1, Array data2, boolean justOne, boolean testTypes) {
    boolean ok = true;
    if (data1.getSize() != data2.getSize()) {
      f.format(" DIFF %s: size %d !== %d%n", name, data1.getSize(), data2.getSize());
      ok = false;
    }

    if (testTypes && data1.getElementType() != data2.getElementType()) {
      f.format(" DIFF %s: element type %s !== %s%n", name, data1.getElementType(), data2.getElementType());
      ok = false;
    }

    if (testTypes && data1.getDataType() != data2.getDataType()) {
      f.format(" DIFF %s: data type %s !== %s%n", name, data1.getDataType(), data2.getDataType());
      ok = false;
    }

    if (!ok) return false;

    DataType dt = data1.getDataType();

    IndexIterator iter1 = data1.getIndexIterator();
    IndexIterator iter2 = data2.getIndexIterator();

    if (data1.isVlen()) {
      while (iter1.hasNext() && iter2.hasNext()) {
        Object v1 = iter1.getObjectNext();
        Object v2 = iter2.getObjectNext();
        if (v1.getClass() != v2.getClass()) {
          f.format(" DIFF %s: ArrayObject class %s != %s %n", name, v1.getClass().getName(), v2.getClass().getName());
          ok = false;
          if (justOne) break;

        } else if (v1 instanceof Array) {
          ok &= compareData(name, (Array) v1, (Array) v2, justOne, testTypes);
        }
      }

    } else if (dt == DataType.DOUBLE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        double v1 = iter1.getDoubleNext();
        double v2 = iter2.getDoubleNext();
        if (!Misc.nearlyEquals(v1, v2)) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt == DataType.FLOAT) {
      while (iter1.hasNext() && iter2.hasNext()) {
        float v1 = iter1.getFloatNext();
        float v2 = iter2.getFloatNext();
        if (!Misc.nearlyEquals(v1, v2)) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt.getPrimitiveClassType() == int.class) {
      while (iter1.hasNext() && iter2.hasNext()) {
        int v1 = iter1.getIntNext();
        int v2 = iter2.getIntNext();
        if (v1 != v2) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt.getPrimitiveClassType() == short.class) {
      while (iter1.hasNext() && iter2.hasNext()) {
        short v1 = iter1.getShortNext();
        short v2 = iter2.getShortNext();
        if (v1 != v2) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt.getPrimitiveClassType() == byte.class) {
      while (iter1.hasNext() && iter2.hasNext()) {
        byte v1 = iter1.getByteNext();
        byte v2 = iter2.getByteNext();
        if (v1 != v2) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt.getPrimitiveClassType() == long.class) {
      while (iter1.hasNext() && iter2.hasNext()) {
        long v1 = iter1.getLongNext();
        long v2 = iter2.getLongNext();
        if (v1 != v2) {
          f.format(createNumericDataDiffMessage(dt, name, v1, v2, iter1));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt.getPrimitiveClassType() == char.class) {
      while (iter1.hasNext() && iter2.hasNext()) {
        char v1 = iter1.getCharNext();
        char v2 = iter2.getCharNext();
        if (v1 != v2) {
          f.format(" DIFF char %s: %s != %s count=%s%n", name, v1, v2, iter1);
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt == DataType.STRING) {
      while (iter1.hasNext() && iter2.hasNext()) {
        String v1 = (String) iter1.getObjectNext();
        String v2 = (String) iter2.getObjectNext();
        if (!v1.equals(v2)) {
          f.format(" DIFF string %s: %s != %s count=%s%n", name, v1, v2, iter1);
          ok = false;
          if (justOne) break;
        }
      }

    } else if (dt == DataType.STRUCTURE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        compareStructureData((StructureData) iter1.next(), (StructureData) iter2.next(), justOne);
      }

    } else if (dt == DataType.OPAQUE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        ByteBuffer obj1 = (ByteBuffer) iter1.next();
        ByteBuffer obj2 = (ByteBuffer) iter2.next();
        if (obj1.limit() != obj2.limit()) {
          f.format(" DIFF %s: opaque size %d != %d%n", name, obj1.limit(), obj2.limit() );
          ok = false;
          if (justOne) break;
        }
      }

    } else {
      ok = false;
      f.format(" %s: Unknown data type %s%n", name, data1.getClass().getName());
    }

    return ok;
  }

  private String createNumericDataDiffMessage(DataType dt, String name, Number v1, Number v2, IndexIterator iter) {
    return String.format(" DIFF %s %s: %s != %s;  count = %s, absDiff = %s, relDiff = %s %n",
            dt, name, v1, v2, iter, Misc.absoluteDifference(v1.doubleValue(), v2.doubleValue()),
            Misc.relativeDifference(v1.doubleValue(), v2.doubleValue()));
  }

  public boolean compareStructureData(StructureData sdata1, StructureData sdata2, boolean justOne) {
    boolean ok = true;

    StructureMembers sm1 = sdata1.getStructureMembers();
    StructureMembers sm2 = sdata2.getStructureMembers();
    if (sm1.getMembers().size() != sm2.getMembers().size()) {
      f.format(" size %d !== %d%n", sm1.getMembers().size(), sm2.getMembers().size());
      ok = false;
    }

    for (StructureMembers.Member m1 : sm1.getMembers()) {
      // if (m1.getName().equals("time")) continue;
      StructureMembers.Member m2 = sm2.findMember(m1.getName());
      Array data1 = sdata1.getArray(m1);
      Array data2 = sdata2.getArray(m2);
      ok &= compareData(m1.getName(), data1, data2, justOne, true);
    }

    return ok;
  }

  public static void main(String arg[]) throws IOException {
    String usage = "usage: ucar.nc2.util.CompareNetcdf2 file1 file2 [-showEach] [-compareData]";
    if (arg.length < 2) {
      System.out.println(usage);
      System.exit(0);
    }

    boolean showEach = false;
    boolean compareData = false;

    String file1 = arg[0];
    String file2 = arg[1];

    for (int i = 2; i < arg.length; i++) {
      String s = arg[i];
      if (s.equalsIgnoreCase("-showEach")) showEach = true;
      if (s.equalsIgnoreCase("-compareData")) compareData = true;
    }

    NetcdfFile ncfile1 = NetcdfDataset.open(file1);
    NetcdfFile ncfile2 = NetcdfDataset.open(file2);
    compareFiles(ncfile1, ncfile2, new Formatter(System.out), true, compareData, showEach);
    ncfile1.close();
    ncfile2.close();
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.dataset.conv;

import ucar.nc2.Attribute;
import ucar.nc2.constants.CF;
import ucar.nc2.dataset.*;
import ucar.nc2.time.Calendar;
import ucar.nc2.units.SimpleUnit;
import ucar.nc2.Variable;
import ucar.nc2.constants.AxisType;
import ucar.nc2.util.CancelTask;

import java.io.IOException;
import java.util.List;

/**
 * COARDS Convention.
 * see http://ferret.wrc.noaa.gov/noaa_coop/coop_cdf_profile.html
 *
 * @author caron
 */

public class COARDSConvention extends CoordSysBuilder {

  /*
  The COARDS standard offers limited support for climatological time. For compatibility with COARDS, time coordinates should also be
  recognised as climatological if they have a units attribute of time-units relative to midnight on 1 January in year 0
  i.e. since 0-1-1 in udunits syntax , and provided they refer to the real-world calendar. We do not recommend this convention because
   (a) it does not provide any information about the intervals used to compute the climatology, and
   (b) there is no standard for how dates since year 1 will be encoded with units having a reference time in year 0,
  since this year does not exist; consequently there may be inconsistencies among software packages in the interpretation of the
  time coordinates. Year 0 may be a valid year in non-real-world calendars, and therefore cannot be used to signal climatological
  time in such cases.
   */

  public static boolean isMine(String hasName) {
    if (hasName.equalsIgnoreCase("COARDS")) return true;
    List<String> names = breakupConventionNames(hasName);
    for (String name : names) {
      if (name.equalsIgnoreCase("COARDS")) return true;
    }
    return false;
  }

  public COARDSConvention() {
    this.conventionName = "COARDS";
  }

  protected boolean checkTimeVarForCalendar (Variable var) {
    boolean hasChanged = false;
    String unit = var.getUnitsString();
    if (unit != null) {
      unit = unit.trim();
      if (SimpleUnit.isDateUnit(unit)) {
        Attribute calAttr = var.findAttributeIgnoreCase(CF.CALENDAR);
        if (calAttr == null) {
          calAttr = new Attribute(CF.CALENDAR, Calendar.gregorian.toString());
          var.addAttribute(calAttr);
          hasChanged = true;
        }
      }
    }
    return hasChanged;
  }

  @Override
  public void augmentDataset(NetcdfDataset ds, CancelTask cancelTask) throws IOException {
    boolean hasChanged = false;
    boolean hasChangedThisTime;
    for (Variable var : ds.getVariables()) {
      hasChangedThisTime = checkTimeVarForCalendar(var);
      if (hasChangedThisTime) {
        hasChanged = true;
      }
    }
    if (hasChanged) {
      ds.finish();
    }
  }

  protected boolean checkForMeter = true;

  // we assume that coordinate axes get identified by being coordinate variables
  protected AxisType getAxisType( NetcdfDataset ncDataset, VariableEnhanced v) {

    String unit = v.getUnitsString();
    if (unit == null)
      return null;
    unit = unit.trim();

    if( unit.equalsIgnoreCase("degrees_east") ||
            unit.equalsIgnoreCase("degrees_E") ||
            unit.equalsIgnoreCase("degreesE") ||
            unit.equalsIgnoreCase("degree_east") ||
            unit.equalsIgnoreCase("degree_E") ||
            unit.equalsIgnoreCase("degreeE"))
      return AxisType.Lon;

    if ( unit.equalsIgnoreCase("degrees_north") ||
            unit.equalsIgnoreCase("degrees_N") ||
            unit.equalsIgnoreCase("degreesN") ||
            unit.equalsIgnoreCase("degree_north") ||
            unit.equalsIgnoreCase("degree_N") ||
            unit.equalsIgnoreCase("degreeN"))
      return AxisType.Lat;

    if (SimpleUnit.isDateUnit(unit)) {
      return AxisType.Time;
    }
    // look for other z coordinate
    if (SimpleUnit.isCompatible("mbar", unit))
      return AxisType.Pressure;
    if (unit.equalsIgnoreCase("level") || unit.equalsIgnoreCase("layer") || unit.equalsIgnoreCase("sigma_level"))
      return AxisType.GeoZ;

    String positive = ncDataset.findAttValueIgnoreCase((Variable) v, CF.POSITIVE, null);
    if (positive != null) {
      if (SimpleUnit.isCompatible("m", unit))
        return AxisType.Height;
      else
        return AxisType.GeoZ;
    }

    // a bad idea, but CDC SST relies on it :
    // :Source = "NOAA/National Climatic Data Center";
   // :Contact = "Dick Reynolds, email: Richard.W.Reynolds@noaa.gov & Chunying Liu, email: Chunying.liu@noaa.gov";
   //:netcdf_Convention = "COARDS";
   // if (checkForMeter && SimpleUnit.isCompatible("m", unit))
   //   return AxisType.Height;

    return null;
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib1.tables;

import javax.annotation.Nullable;
import ucar.nc2.grib.GribStatType;

/**
 * Standard WMO tables for time range indicator - Grib1 table 5.
 *
 * @author caron
 * @since 1/13/12
 */
public class Grib1WmoTimeType {

  /**
   * The time unit statistical type, derived from code table 5)
   *
   * @return time unit statistical type, or null if unknown.
   */
  @Nullable
  public static GribStatType getStatType(int timeRangeIndicator) {
    switch (timeRangeIndicator) {
      case 3:
      case 6:
      case 7:
      case 51:
      case 113:
      case 115:
      case 117:
      case 120:
      case 123:
        return GribStatType.Average;
      case 4:
      case 114:
      case 116:
      case 124:
        return GribStatType.Accumulation;
      case 5:
        return GribStatType.DifferenceFromEnd;
      case 118:
        return GribStatType.Covariance;
      case 119:
      case 125:
        return GribStatType.StandardDeviation;
      default:
        return null;
    }
  }

}