/*
 * Copyright (c) 1998-2017 University Corporation for Atmospheric Research/Unidata
 */
package ucar.nc2.dataset;

import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.CF;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.constants.AxisType;
import ucar.nc2.util.CancelTask;
import ucar.nc2.ncml.NcMLReader;
import ucar.nc2.dataset.conv.*;
import ucar.ma2.DataType;

import javax.annotation.Nonnull;
import java.lang.reflect.Method;
import java.io.IOException;
import java.util.*;

/**
 * Abstract class for implementing Convention-specific parsing of netCDF files.
 *

 * You can use an NcML file alone (use registerNcML()) if file uses a convention attribute.
 * If not, you must implement a class that implements isMine() to identify your files, and
 * call wrapNcML in the augmentDataset method (see eg ATDRadarConvention class).
 *

 *

 * Subclasses Info:
 *

 * // identify which variables are coordinate axes
 * // default: 1) coordinate variables 2) variables with _coordinateAxisType attribute 3) variables listed
 * // in a coordinates attribute on another variable.
 * findCoordinateAxes( ncDataset);
 *
 * // identify which variables are used to describe coordinate system
 * findCoordinateSystems( ncDataset);
 * // identify which variables are used to describe coordinate transforms
 * findCoordinateTransforms( ncDataset);
 * // turn Variables into CoordinateAxis objects
 * makeCoordinateAxes( ncDataset);
 * // make Coordinate Systems for all Coordinate Systems Variables
 * makeCoordinateSystems( ncDataset);
 *
 * // Assign explicit CoordinateSystem objects to variables
 * assignExplicitCoordinateSystems( ncDataset);
 * makeCoordinateSystemsImplicit( ncDataset);
 * if (useMaximalCoordSys)
 * makeCoordinateSystemsMaximal( ncDataset);
 *
 * makeCoordinateTransforms( ncDataset);
 * assignCoordinateTransforms( ncDataset);
 *

 *
 * @author caron
 */

/*
  Implementation notes:

  Generally, subclasses should add the _Coordinate conventions, see
    http://www.unidata.ucar.edu/software/netcdf-java/CoordinateAttributes.html
  Then let this class do the rest of the work.

  How to add Coordinate Transforms:
    A.
    1) create a dummy Variable called the Coordinate Transform Variable.
      This Coordinate Transform variable always has a name that identifies the transform,
      and any attributes needed for the transformation.
    2) explicitly point to it by adding a _CoordinateTransform attribute to a Coordinate System Variable
       _CoordinateTransforms = "LambertProjection HybridSigmaVerticalTransform"

    B. You could explicitly add it by overriding assignCoordinateTransforms()
 */

public class CoordSysBuilder implements CoordSysBuilderIF {
  static public final String resourcesDir = "resources/nj22/coords/"; // resource path
  static protected org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(CoordSysBuilder.class);

  static private List<Convention> conventionList = new ArrayList<>();
  static private Map<String, String> ncmlHash = new HashMap<>();
  static private boolean useMaximalCoordSys = true;
  static private boolean userMode = false;

  /**
   * Allow plug-ins to determine if it owns a file based on the file's Convention attribute.
   */
  public interface ConventionNameOk {

    /**
     * Do you own this file?
     *
     * @param convName name in the file
     * @param wantName name passed into registry
     * @return true if you own this file
     */
    boolean isMatch(String convName, String wantName);
  }

  // search in the order added
  static { // wont get loaded unless explicitly called
    // ours
    registerConvention(_Coordinate.Convention, CoordSysBuilder.class, null);
    registerConvention("CF-1.", CF1Convention.class, (convName, wantName) -> {
      if (convName.startsWith(wantName)) return true;
      List<String> names = breakupConventionNames(convName);
      for (String name : names)
        if (name.startsWith(wantName)) return true;
      return false;
    });
    registerConvention("CDM-Extended-CF", CF1Convention.class);

    registerConvention("COARDS", COARDSConvention.class, null);
    registerConvention("NCAR-CSM", CSMConvention.class, null);
    registerConvention("Unidata Observation Dataset v1.0", UnidataObsConvention.class, null);
    registerConvention("GDV", GDVConvention.class, null);

    registerConvention("ATDRadar", ATDRadarConvention.class, null);
    registerConvention("CEDRICRadar", CEDRICRadarConvention.class, null);
    registerConvention("Zebra", ZebraConvention.class, null);
    registerConvention("GIEF/GIEF-F", GIEFConvention.class, null);
    registerConvention("IRIDL", IridlConvention.class, null);

    // the uglies
    registerConvention("NUWG", NUWGConvention.class, null);
    registerConvention("AWIPS", AWIPSConvention.class, null);
    registerConvention("AWIPS-Sat", AWIPSsatConvention.class, null);
    registerConvention("WRF", WRFConvention.class, null);

    //registerConvention("M3IOVGGrid", M3IOVGGridConvention.class, null);
    registerConvention("M3IO", M3IOConvention.class, null);
    registerConvention("IFPS", IFPSConvention.class, null);
    registerConvention("ARPS/ADAS", ADASConvention.class, null);

    // point data
    registerConvention("MADIS surface observations, v1.0", MADISStation.class, null);
    registerConvention("epic-insitu-1.0", EpicInsitu.class, null);
    registerConvention("NCAR-RAF/nimbus", Nimbus.class, null);
    registerConvention("Cosmic1Convention", Cosmic1Convention.class, null);
    registerConvention("Jason2Convention", Jason2Convention.class, null);
    registerConvention("Suomi", Suomi.class, null);

    // new
    registerConvention("NSSL National Reflectivity Mosaic", NsslRadarMosaicConvention.class, null);
    registerConvention("FslWindProfiler", FslWindProfiler.class, null);
    registerConvention("ModisSatellite", ModisSatellite.class, null);
    registerConvention("AvhrrSatellite", AvhrrConvention.class, null);
    registerConvention("NPP/NPOESS", NppConvention.class, null);
    registerConvention("HDF5-EOS-OMI", HdfEosOmiConvention.class, null);
    registerConvention("HDF4-EOS-MODIS", HdfEosModisConvention.class, null);

    // further calls to registerConvention are by the user
    userMode = true;
  }

  /**
   * Register an NcML file that implements a Convention by wrappping the dataset in the NcML.
   * It is then processed by CoordSysBuilder, using the _Coordinate attributes.
   *
   * @param conventionName name of Convention, must be in the "Conventions" global attribute.
   * @param ncmlLocation   location of NcML file, may be local file or URL.
   * @see ucar.nc2.ncml.NcMLReader#wrapNcML
   */
  static public void registerNcML(String conventionName, String ncmlLocation) {
    ncmlHash.put(conventionName, ncmlLocation);
  }

  /**
   * Register a class that implements a Convention. Will match (ignoring case) the COnvention name.
   *
   * @param conventionName name of Convention.
   *                       This name will be used to look in the "Conventions" global attribute.
   *                       Otherwise, you must implement the isMine() static method.
   * @param c              implementation of CoordSysBuilderIF that parses those kinds of netcdf files.
   */
  static public void registerConvention(String conventionName, Class c) {
    registerConvention(conventionName, c, null);
  }

  /**
   * Register a class that implements a Convention.
   *
   * @param conventionName name of Convention.
   *                       This name will be used to look in the "Conventions" global attribute.
   *                       Otherwise, you must implement the isMine() static method.
   * @param match          pass in your own matcher. if null, equalsIgnoreCase() will be used.
   * @param c              implementation of CoordSysBuilderIF that parses those kinds of netcdf files.
   */
  static public void registerConvention(String conventionName, Class c, ConventionNameOk match) {

    if (!(CoordSysBuilderIF.class.isAssignableFrom(c)))
      throw new IllegalArgumentException("CoordSysBuilderIF Class " + c.getName() + " must implement CoordSysBuilderIF");

    // fail fast - check newInstance works
    try {
      c.newInstance();
    } catch (InstantiationException e) {
      throw new IllegalArgumentException("CoordSysBuilderIF Class " + c.getName() + " cannot instantiate, probably need default Constructor");
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("CoordSysBuilderIF Class " + c.getName() + " is not accessible");
    }

    // user stuff gets put at top
    if (userMode)
      conventionList.add(0, new Convention(conventionName, c, match));
    else
      conventionList.add(new Convention(conventionName, c, match));
  }

  static private Class matchConvention(String convName) {
    for (Convention c : conventionList) {
      if ((c.match == null) && c.convName.equalsIgnoreCase(convName)) return c.convClass;
      if ((c.match != null) && c.match.isMatch(convName, c.convName)) return c.convClass;
    }
    return null;
  }

  /**
   * Register a class that implements a Convention.
   *
   * @param conventionName name of Convention.
   *                       This name will be used to look in the "Conventions" global attribute.
   *                       Otherwise, you must implement the isMine() static method.
   * @param className      name of class that implements CoordSysBuilderIF.
   * @throws ClassNotFoundException if class could not be loaded
   */
  static public void registerConvention(String conventionName, String className) throws ClassNotFoundException {
    Class c = Class.forName(className);
    registerConvention(conventionName, c, null);
  }

  /**
   * If true, assign implicit CoordinateSystem objects to variables that dont have one yet.
   * Default value is false.
   *
   * @param b true if if you want to guess at Coordinate Systems
   * @see #makeCoordinateSystemsMaximal
   */
  static public void setUseMaximalCoordSys(boolean b) {
    useMaximalCoordSys = b;
  }

  /**
   * Get whether to make records into Structures.
   *
   * @return whether to make records into Structures.
   */
  static public boolean getUseMaximalCoordSys() {
    return useMaximalCoordSys;
  }

  /**
   * Breakup list of Convention names in the Convention attribute in CF compliant way.
   *
   * @param convAttValue original value of Convention attribute
   * @return list of Convention names
   */
  static public List<String> breakupConventionNames(String convAttValue) {
    List<String> names = new ArrayList<>();

    if ((convAttValue.indexOf(',') > 0) || (convAttValue.indexOf(';') > 0)) {
      StringTokenizer stoke = new StringTokenizer(convAttValue, ",;");
      while (stoke.hasMoreTokens()) {
        String name = stoke.nextToken();
        names.add(name.trim());
      }
    } else if ((convAttValue.indexOf('/') > 0)) {
      StringTokenizer stoke = new StringTokenizer(convAttValue, "/");
      while (stoke.hasMoreTokens()) {
        String name = stoke.nextToken();
        names.add(name.trim());
      }
    } else {
      StringTokenizer stoke = new StringTokenizer(convAttValue, " ");
      while (stoke.hasMoreTokens()) {
        String name = stoke.nextToken();
        names.add(name.trim());
      }
    }
    return names;
  }

  /**
   * Build a list of Conventions
   *
   * @param mainConv this is the main convention
   * @param convAtts list of others, onbly use "extra" Conventions
   * @return comma separated list of Conventions
   */
  static public String buildConventionAttribute(String mainConv, String... convAtts) {
    List<String> result = new ArrayList<>();
    result.add(mainConv);
    for (String convs : convAtts) {
      if (convs == null) continue;
      List<String> ss = breakupConventionNames(convs); // may be a list
      for (String s : ss) {
        if (matchConvention(s) == null)  // only add extra ones, not ones that compete with mainConv
          result.add(s);
      }
    }

    // now form comma separated result
    boolean start = true;
    Formatter f = new Formatter();
    for (String s : result) {
      if (start)
        f.format("%s", s);
      else
        f.format(", %s", s);
      start = false;
    }
    return f.toString();
  }

  /**
   * Get a CoordSysBuilder whose job it is to add Coordinate information to a NetcdfDataset.
   *
   * @param ds         the NetcdfDataset to modify
   * @param cancelTask allow user to bail out.
   * @return the builder used
   * @throws java.io.IOException on io error
   * @see ucar.nc2.dataset.NetcdfDataset#enhance
   */
  static public @Nonnull
  CoordSysBuilderIF factory(NetcdfDataset ds, CancelTask cancelTask) throws IOException {

    // look for the Conventions attribute
    String convName = ds.findAttValueIgnoreCase(null, CDM.CONVENTIONS, null);
    if (convName == null)
      convName = ds.findAttValueIgnoreCase(null, "Convention", null); // common mistake Convention instead of Conventions
    if (convName != null)
      convName = convName.trim();

    // look for ncml first
    if (convName != null) {
      String convNcML = ncmlHash.get(convName);
      if (convNcML != null) {
        CoordSysBuilder csb = new CoordSysBuilder();
        NcMLReader.wrapNcML(ds, convNcML, cancelTask);
        return csb;
      }
    }

    // look for registered conventions using convention name
    Class convClass = null;
    if (convName != null) {
      convClass = matchConvention(convName);

      // now look for comma or semicolon or / delimited list
      if (convClass == null) {
        List<String> names = breakupConventionNames(convName);
        if (names.size() > 0) {
          // search the registered conventions, in order
          for (Convention conv : conventionList) {
            for (String name : names) {
              if (name.equalsIgnoreCase(conv.convName)) {
                convClass = conv.convClass;
                convName = name;
              }
            }
            if (convClass != null) break;
          }
        }
      }
    }

    // look for ones that dont use Convention attribute, in order added.
    // call static isMine() using reflection.
    if (convClass == null) {
      for (Convention conv : conventionList) {
        Class c = conv.convClass;
        Method m;

        try {
          m = c.getMethod("isMine", NetcdfFile.class);   // LOOK cant we test if method exists ?
        } catch (NoSuchMethodException ex) {
          continue;
        }

        try {
          Boolean result = (Boolean) m.invoke(null, ds);
          if (result) {
            convClass = c;
            break;
          }
        } catch (Exception ex) {
          log.error("ERROR: Class " + c.getName() + " Exception invoking isMine method\n" + ex);
        }
      }
    }

    // use service loader mechanism
    // call static isMine() using reflection.
    CoordSysBuilderIF builder = null;
    if (convClass == null) {
      for (CoordSysBuilderIF csb : ServiceLoader.load(CoordSysBuilderIF.class)) {

        Class c = csb.getClass();
        Method m;
        try {
          m = c.getMethod("isMine", NetcdfFile.class);
        } catch (NoSuchMethodException ex) {
          continue;
        }

        try {
          Boolean result = (Boolean) m.invoke(null, ds);
          if (result) {
            builder = csb;
            convClass = c;
            break;
          }
        } catch (Exception ex) {
          log.error("ERROR: Class " + c.getName() + " Exception invoking isMine method%n" + ex);
        }
      }

    }

    // if no convention class found, use the default
    if (convClass == null)
      convClass = DefaultConvention.class;

    if (builder == null) {
      // get an instance of the class
      try {
        builder = (CoordSysBuilderIF) convClass.newInstance();
      } catch (Exception e) {
        log.error("failed on CoordSysBuilderIF for " + convClass.getName(), e);
        throw new RuntimeException(e);
      }
    }

    if (convName == null)
      builder.addUserAdvice("No 'Conventions' global attribute.");
    else if (convClass == DefaultConvention.class)
      builder.addUserAdvice("No CoordSysBuilder is defined for Conventions= '" + convName + "'\n");
    else
      builder.setConventionUsed(convClass.getName());

    ds.addAttribute(null, new Attribute(_Coordinate._CoordSysBuilder, convClass.getName()));
    return builder;
  }

  static private class Convention {
    String convName;
    Class convClass;
    ConventionNameOk match;

    Convention(String convName, Class convClass, ConventionNameOk match) {
      this.convName = convName;
      this.convClass = convClass;
      this.match = match;
    }
  }

  ////////////////////////////////////////////////////////////////////////////////////////

  protected String conventionName = _Coordinate.Convention; // default name of Convention, override in subclass
  protected List<VarProcess> varList = new ArrayList<>(); // varProcess objects
  protected Map<Dimension, List<VarProcess>> coordVarMap = new HashMap<>();
  protected Formatter parseInfo = new Formatter();
  protected Formatter userAdvice = new Formatter();

  protected boolean debug = false;

  @Override
  public void setConventionUsed(String convName) {
    this.conventionName = convName;
  }

  @Override
  public String getConventionUsed() {
    return conventionName;
  }

  @Override
  public void addUserAdvice(String advice) {
    userAdvice.format("%s", advice);
  }

  @Override
  public String getParseInfo() {
    return parseInfo.toString();
  }

  @Override
  public String getUserAdvice() {
    return userAdvice.toString();
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////
  // subclasses can override any of these routines

  @Override
  public void augmentDataset(NetcdfDataset ncDataset, CancelTask cancelTask) throws IOException {
  }

  /**
   * Identify what kind of AxisType the named variable is.
   * Only called for variables already identified as Coordinate Axes.
   * Default null - subclasses can override.
   *
   * @param ncDataset for this dataset
   * @param v         a variable alreaddy identified as a Coodinate Axis
   * @return AxisType or null if unknown.
   */
  protected AxisType getAxisType(NetcdfDataset ncDataset, VariableEnhanced v) {
    return null;
  }

  /**
   * Heres where the work is to identify coordinate axes and coordinate systems.
   *
   * @param ncDataset modify this dataset
   */
  @Override
  public void buildCoordinateSystems(NetcdfDataset ncDataset) {
    // put status info into parseInfo that can be shown to someone trying to debug this process
    parseInfo.format("Parsing with Convention = %s%n", conventionName);

    // Bookkeeping info for each variable is kept in the VarProcess inner class
    addVariables(ncDataset, ncDataset.getVariables(), varList);

    // identify which variables are coordinate axes
    findCoordinateAxes(ncDataset);
    // identify which variables are used to describe coordinate system
    findCoordinateSystems(ncDataset);
    // identify which variables are used to describe coordinate transforms
    findCoordinateTransforms(ncDataset);
    // turn Variables into CoordinateAxis objects
    makeCoordinateAxes(ncDataset);
    // make Coordinate Systems for all Coordinate Systems Variables
    makeCoordinateSystems(ncDataset);

    // assign explicit CoordinateSystem objects to variables
    assignCoordinateSystemsExplicit(ncDataset);

    // assign implicit CoordinateSystem objects to variables
    makeCoordinateSystemsImplicit(ncDataset);

    // optionally assign implicit CoordinateSystem objects to variables that dont have one yet
    if (useMaximalCoordSys)
      makeCoordinateSystemsMaximal(ncDataset);

    // make Coordinate Transforms
    makeCoordinateTransforms(ncDataset);

    // assign Coordinate Transforms
    assignCoordinateTransforms(ncDataset);

    if (debug) System.out.println("parseInfo = \n" + parseInfo.toString());
  }

  private void addVariables(NetcdfDataset ncDataset, List<Variable> varList, List<VarProcess> varProcessList) {
    for (Variable v : varList) {
      varProcessList.add(new VarProcess(ncDataset, v));

      if (v instanceof Structure) {
        List<Variable> nested = ((Structure) v).getVariables();
        addVariables(ncDataset, nested, varProcessList);
      }
    }
  }

  /**
   * Identify coordinate axes, set VarProcess.isCoordinateAxis = true.
   * Default is to look for those referenced by _CoordinateAxes attribute.
   * Note coordinate variables are already identified.
   *
   * @param ncDataset why
   */
  protected void findCoordinateAxes(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.coordAxes != null)
        findCoordinateAxes(vp, vp.coordAxes);
      if (vp.coordinates != null)
        findCoordinateAxes(vp, vp.coordinates);
    }
  }

  private void findCoordinateAxes(VarProcess vp, String coordinates) {
    StringTokenizer stoker = new StringTokenizer(coordinates);
    while (stoker.hasMoreTokens()) {
      String vname = stoker.nextToken();
      VarProcess ap = findVarProcess(vname, vp);
      if (ap == null) {
        Group g = vp.v.getParentGroup();
        Variable v = g.findVariableOrInParent(vname);
        if (v != null)
          ap = findVarProcess(v.getFullName(), vp);
        else {
          parseInfo.format("***Cant find coordAxis %s referenced from var= %s%n", vname, vp.v.getFullName());
          userAdvice.format("***Cant find coordAxis %s referenced from var= %s%n", vname, vp.v.getFullName());
        }
      }

      if (ap != null) {
        if (!ap.isCoordinateAxis)
          parseInfo.format(" CoordinateAxis = %s added; referenced from var= %s%n", vname, vp.v.getFullName());
        ap.isCoordinateAxis = true;
      } else {
        parseInfo.format("***Cant find coordAxis %s referenced from var= %s%n", vname, vp.v.getFullName());
        userAdvice.format("***Cant find coordAxis %s referenced from var= %s%n", vname, vp.v.getFullName());
      }
    }
  }

  /**
   * Identify coordinate systems, set VarProcess.isCoordinateSystem = true.
   * Default is to look for those referenced by _CoordinateSystems attribute.
   *
   * @param ncDataset why
   */
  protected void findCoordinateSystems(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.coordSys != null) {
        StringTokenizer stoker = new StringTokenizer(vp.coordSys);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, vp);
          if (ap != null) {
            if (!ap.isCoordinateSystem)
              parseInfo.format(" CoordinateSystem = %s added; referenced from var= %s%n", vname, vp.v.getFullName());
            ap.isCoordinateSystem = true;
          } else {
            parseInfo.format("***Cant find coordSystem %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Cant find coordSystem %s referenced from var= %s%n", vname, vp.v.getFullName());
          }
        }
      }
    }
  }

  /**
   * Identify coordinate transforms, set VarProcess.isCoordinateTransform = true.
   * Default is to look for those referenced by _CoordinateTransforms attribute
   * ( or has a _CoordinateTransformType attribute, done in VarProcess constructor)
   *
   * @param ncDataset why
   */
  protected void findCoordinateTransforms(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.coordTransforms != null) {
        StringTokenizer stoker = new StringTokenizer(vp.coordTransforms);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, vp);
          if (ap != null) {
            if (!ap.isCoordinateTransform)
              parseInfo.format(" CoordinateTransform = %s added; referenced from var= %s%n", vname, vp.v.getFullName());
            ap.isCoordinateTransform = true;
          } else {
            parseInfo.format("***Cant find CoordinateTransform %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Cant find CoordinateTransform %s referenced from var= %s%n", vname, vp.v.getFullName());
          }
        }
      }
    }
  }

  /**
   * Take previously identified Coordinate Axis and Coordinate Variables and make them into a
   * CoordinateAxis. Uses the getAxisType() method to figure out the type, if not already set.
   *
   * @param ncDataset containing dataset
   */
  protected void makeCoordinateAxes(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.isCoordinateAxis || vp.isCoordinateVariable) {
        if (vp.axisType == null)
          vp.axisType = getAxisType(ncDataset, (VariableEnhanced) vp.v);
        if (vp.axisType == null) {
          userAdvice.format("Coordinate Axis %s does not have an assigned AxisType%n", vp.v.getFullName());
        }
        vp.makeIntoCoordinateAxis();
      }
    }
  }

  /**
   * Take all previously identified Coordinate Systems and create a
   * CoordinateSystem object.
   *
   * @param ncDataset why
   */
  protected void makeCoordinateSystems(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.isCoordinateSystem) {
        vp.makeCoordinateSystem();
      }
    }
  }

  /**
   * Assign explicit CoordinateSystem objects to variables.
   *
   * @param ncDataset why
   */
  protected void assignCoordinateSystemsExplicit(NetcdfDataset ncDataset) {

    // look for explicit references to coord sys variables
    for (VarProcess vp : varList) {
      if (vp.coordSys != null && !vp.isCoordinateTransform) {
        StringTokenizer stoker = new StringTokenizer(vp.coordSys);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, vp);
          if (ap == null) {
            parseInfo.format("***Cant find Coordinate System variable %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Cant find Coordinate System variable %s referenced from var= %s%n", vname, vp.v.getFullName());
            continue;
          }
          if (ap.cs == null) {
            parseInfo.format("***Not a Coordinate System variable %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Not a Coordinate System variable %s referenced from var= %s%n", vname, vp.v.getFullName());
            continue;
          }

          VariableEnhanced ve = (VariableEnhanced) vp.v;
          ve.addCoordinateSystem(ap.cs);
        }
      }
    }

    // look for explicit references from coord sys variables to data variables
    for (VarProcess csVar : varList) {
      if (!csVar.isCoordinateSystem || (csVar.coordSysFor == null))
        continue;

      // get list of dimensions from '_CoordinateSystemFor' attribute
      List<Dimension> dimList = new ArrayList<>(6);
      StringTokenizer stoker = new StringTokenizer(csVar.coordSysFor);
      while (stoker.hasMoreTokens()) {
        String dname = stoker.nextToken();
        Dimension dim = ncDataset.getRootGroup().findDimension(dname);
        if (dim == null) {
          parseInfo.format("***Cant find Dimension %s referenced from CoordSys var= %s%n", dname, csVar.v.getFullName());
          userAdvice.format("***Cant find Dimension %s referenced from CoordSys var= %s%n", dname, csVar.v.getFullName());
        } else
          dimList.add(dim);
      }

      // look for vars with those dimensions
      for (VarProcess vp : varList) {
        if (!vp.hasCoordinateSystem() && vp.isData() && (csVar.cs != null)) {
          VariableEnhanced ve = (VariableEnhanced) vp.v;
          if (CoordinateSystem.isSubset(dimList, vp.v.getDimensionsAll()) && CoordinateSystem.isSubset(vp.v.getDimensionsAll(), dimList))
            ve.addCoordinateSystem(csVar.cs);
        }
      }
    }

    // look for explicit listings of coordinate axes
    for (VarProcess vp : varList) {
      VariableEnhanced ve = (VariableEnhanced) vp.v;

      if (!vp.hasCoordinateSystem() && (vp.coordAxes != null) && vp.isData()) {
        List<CoordinateAxis> dataAxesList = getAxes(vp, vp.coordAxes, vp.v.getFullName());
        if (dataAxesList.size() > 1) {
          String coordSysName = CoordinateSystem.makeName(dataAxesList);
          CoordinateSystem cs = ncDataset.findCoordinateSystem(coordSysName);
          if (cs != null) {
            ve.addCoordinateSystem(cs);
            parseInfo.format(" assigned explicit CoordSystem '%s' for var= %s%n", cs.getName(), vp.v.getFullName());
          } else {
            CoordinateSystem csnew = new CoordinateSystem(ncDataset, dataAxesList, null);
            ve.addCoordinateSystem(csnew);
            ncDataset.addCoordinateSystem(csnew);
            parseInfo.format(" created explicit CoordSystem '%s' for var= %s%n", csnew.getName(), vp.v.getFullName());
          }
        }
      }
    }
  }

  private List<CoordinateAxis> getAxes(VarProcess vp, String names, String varName) {
    List<CoordinateAxis> axesList = new ArrayList<>();
    StringTokenizer stoker = new StringTokenizer(names);
    while (stoker.hasMoreTokens()) {
      String vname = stoker.nextToken();
      VarProcess ap = findVarProcess(vname, vp);
      if (ap != null) {
        CoordinateAxis axis = ap.makeIntoCoordinateAxis();
        if (!axesList.contains(axis)) axesList.add(axis);
      } else {
        parseInfo.format("***Cant find Coordinate Axis %s referenced from var= %s%n", vname, varName);
        userAdvice.format("***Cant find Coordinate Axis %s referenced from var= %s%n", vname, varName);
      }
    }
    return axesList;
  }

  /**
   * Make implicit CoordinateSystem objects for variables that dont already have one, by using the
   * variables' list of coordinate axes, and any coordinateVariables for it. Must be at least 2 axes.
   * All of a variable's _Coordinate Variables_ plus any variables listed in a *__CoordinateAxes_* or *_coordinates_* attribute
   * will be made into an *_implicit_* Coordinate System.
   * If there are at least two axes, and the coordinate system uses all of the variable's dimensions,
   * it will be asssigned to the data variable.
   */
  protected void makeCoordinateSystemsImplicit(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (!vp.hasCoordinateSystem() && vp.maybeData()) {
        List<CoordinateAxis> dataAxesList = vp.findCoordinateAxes(true);
        if (dataAxesList.size() < 2)
          continue;

        VariableEnhanced ve = (VariableEnhanced) vp.v;
        String csName = CoordinateSystem.makeName(dataAxesList);
        CoordinateSystem cs = ncDataset.findCoordinateSystem(csName);
        if ((cs != null) && cs.isComplete(vp.v)) { // must be complete
          ve.addCoordinateSystem(cs);
          parseInfo.format(" assigned implicit CoordSystem '%s' for var= %s%n", cs.getName(), vp.v.getFullName());
        } else {
          CoordinateSystem csnew = new CoordinateSystem(ncDataset, dataAxesList, null);
          csnew.setImplicit(true);
          if (csnew.isComplete(vp.v)) {
            ve.addCoordinateSystem(csnew);
            ncDataset.addCoordinateSystem(csnew);
            parseInfo.format(" created implicit CoordSystem '%s' for var= %s%n", csnew.getName(), vp.v.getFullName());
          }
        }
      }
    }
  }

  /**
   * If a variable still doesnt have a coordinate system, use hueristics to try to find one that was probably forgotten.
   * Examine existing CS. create a subset of axes that fits the variable. Choose the one with highest rank.
   * It must have X,Y or lat,lon. If so, add it.
   *
   * @param ncDataset need this to see if incomplete coordinate systems are allowed
   */
  protected void makeCoordinateSystemsMaximal(NetcdfDataset ncDataset) {

    boolean requireCompleteCoordSys = !ncDataset.getEnhanceMode().contains(NetcdfDataset.Enhance.IncompleteCoordSystems);

    for (VarProcess vp : varList) {
      VariableEnhanced ve = (VariableEnhanced) vp.v;

      if (vp.hasCoordinateSystem() || !vp.isData()) continue;

      // look through all axes that fit
      List<CoordinateAxis> axisList = new ArrayList<>();
      List<CoordinateAxis> axes = ncDataset.getCoordinateAxes();
      for (CoordinateAxis axis : axes) {
        if (isCoordinateAxisForVariable(axis, ve))
          axisList.add(axis);
      }

      if (axisList.size() < 2) continue;

      String csName = CoordinateSystem.makeName(axisList);
      CoordinateSystem cs = ncDataset.findCoordinateSystem(csName);
      boolean okToBuild = false;

      // do coordinate systems need to be complete?
      // default enhance mode is yes, they must be complete
      if (requireCompleteCoordSys) {
        if (cs != null) {
          // only build if coordinate system is complete
          okToBuild = cs.isComplete(ve);
        }
      } else {
        // coordinate system can be incomplete, so we're ok to build if we find something
        okToBuild = true;
      }

      if (cs != null && okToBuild) {
        ve.addCoordinateSystem(cs);
        parseInfo.format(" assigned maximal CoordSystem '%s' for var= %s%n", cs.getName(), ve.getFullName());

      } else {
        CoordinateSystem csnew = new CoordinateSystem(ncDataset, axisList, null);
        // again, do coordinate systems need to be complete?
        // default enhance mode is yes, they must be complete
        if (requireCompleteCoordSys) {
          // only build if new coordinate system is complete
          okToBuild = csnew.isComplete(ve);
        }
        if (okToBuild) {
          csnew.setImplicit(true);
          ve.addCoordinateSystem(csnew);
          ncDataset.addCoordinateSystem(csnew);
          parseInfo.format(" created maximal CoordSystem '%s' for var= %s%n", csnew.getName(), ve.getFullName());
        }
      }
    }
  }

  /**
   * Does this axis "fit" this variable.
   * True if all of the dimensions in the axis also appear in the variable.
   * If char variable, last dimension is left out.
   *
   * @param axis check if this axis is ok for the given variable
   * @param v    the given variable
   * @return true if all of the dimensions in the axis also appear in the variable.
   */
  protected boolean isCoordinateAxisForVariable(Variable axis, VariableEnhanced v) {
    List<Dimension> varDims = v.getDimensionsAll();
    List<Dimension> axisDims = axis.getDimensionsAll();

    // a CHAR variable must really be a STRING, so leave out the last (string length) dimension
    int checkDims = axisDims.size();
    if (axis.getDataType() == DataType.CHAR)
      checkDims--;

    for (int i = 0; i < checkDims; i++) {
      Dimension axisDim = axisDims.get(i);
      if (!varDims.contains(axisDim)) {
        return false;
      }
    }
    return true;
  }

  /**
   * Take all previously identified Coordinate Transforms and create a
   * CoordinateTransform object by calling CoordTransBuilder.makeCoordinateTransform().
   *
   * @param ncDataset why
   */
  protected void makeCoordinateTransforms(NetcdfDataset ncDataset) {
    for (VarProcess vp : varList) {
      if (vp.isCoordinateTransform && vp.ct == null) {
        vp.ct = CoordTransBuilder.makeCoordinateTransform(vp.ds, vp.v, parseInfo, userAdvice);
      }
    }
  }

  protected CoordinateTransform makeCoordinateTransform(NetcdfDataset ds, Variable ctv) {
    return CoordTransBuilder.makeCoordinateTransform(ds, ctv, parseInfo, userAdvice);
  }

  /**
   * Assign CoordinateTransform objects to Coordinate Systems.
   *
   * @param ncDataset why
   */
  protected void assignCoordinateTransforms(NetcdfDataset ncDataset) {

    // look for explicit transform assignments on the coordinate systems
    for (VarProcess vp : varList) {
      if (vp.isCoordinateSystem && vp.coordTransforms != null) {
        StringTokenizer stoker = new StringTokenizer(vp.coordTransforms);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, vp);
          if (ap != null) {
            if (ap.ct != null) {
              vp.addCoordinateTransform(ap.ct);
              parseInfo.format(" assign explicit coordTransform %s to CoordSys= %s%n", ap.ct, vp.cs);
            } else {
              parseInfo.format("***Cant find coordTransform in %s referenced from var= %s%n", vname, vp.v.getFullName());
              userAdvice.format("***Cant find coordTransform in %s referenced from var= %s%n", vname, vp.v.getFullName());
            }
          } else {
            parseInfo.format("***Cant find coordTransform variable= %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Cant find coordTransform variable= %s referenced from var= %s%n", vname, vp.v.getFullName());
          }
        }
      }
    }

    // look for explicit coordSys assignments on the coordinate transforms
    for (VarProcess vp : varList) {
      if (vp.isCoordinateTransform && (vp.ct != null) && (vp.coordSys != null)) {
        StringTokenizer stoker = new StringTokenizer(vp.coordSys);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess vcs = findVarProcess(vname, vp);
          if (vcs == null) {
            parseInfo.format("***Cant find coordSystem variable= %s referenced from var= %s%n", vname, vp.v.getFullName());
            userAdvice.format("***Cant find coordSystem variable= %s referenced from var= %s%n", vname, vp.v.getFullName());
          } else {
            vcs.addCoordinateTransform(vp.ct);
            parseInfo.format("***assign explicit coordTransform %s to CoordSys=  %s%n", vp.ct, vp.cs);
          }
        }
      }
    }

    // look for coordAxes assignments on the coordinate transforms
    for (VarProcess vp : varList) {
      if (vp.isCoordinateTransform && (vp.ct != null) && (vp.coordAxes != null)) {
        List<CoordinateAxis> dataAxesList = vp.findCoordinateAxes(false);

        if (dataAxesList.size() > 0) {
          for (CoordinateSystem cs : ncDataset.getCoordinateSystems()) {
            if (cs.containsAxes(dataAxesList)) {
              cs.addCoordinateTransform(vp.ct);
              parseInfo.format("***assign (implicit coordAxes) coordTransform %s to CoordSys=  %s%n", vp.ct, cs);
            }
          }
        }
      }
    }

    // look for coordAxisType assignments on the coordinate transforms
    for (VarProcess vp : varList) {
      if (vp.isCoordinateTransform && (vp.ct != null) && (vp.coordAxisTypes != null)) {
        List<AxisType> axisTypesList = new ArrayList<>();
        StringTokenizer stoker = new StringTokenizer(vp.coordAxisTypes);
        while (stoker.hasMoreTokens()) {
          String name = stoker.nextToken();
          AxisType atype;
          if (null != (atype = AxisType.getType(name)))
            axisTypesList.add(atype);
        }
        if (axisTypesList.size() > 0) {
          for (CoordinateSystem cs : ncDataset.getCoordinateSystems()) {
            if (cs.containsAxisTypes(axisTypesList)) {
              cs.addCoordinateTransform(vp.ct);
              parseInfo.format("***assign (implicit coordAxisType) coordTransform %s to CoordSys=  %s%n", vp.ct, cs);
            }
          }
        }
      }
    }

  }

  protected VarProcess findVarProcess(String name, VarProcess from) {
    if (name == null) return null;

    // compare full name
    for (VarProcess vp : varList) {
      if (name.equals(vp.v.getFullName()))
        return vp;
    }

    // prefer ones in the same group
    if (from != null) {
      for (VarProcess vp : varList) {
        if (name.equals(vp.v.getShortName()) && vp.v.getGroup().equals(from.v.getGroup()))
          return vp;
      }
    }

    // WAEF, use short name
    for (VarProcess vp : varList) {
      if (name.equals(vp.v.getShortName()))
        return vp;
    }

    return null;
  }

  protected VarProcess findCoordinateAxis(String name) {
    if (name == null) return null;

    for (VarProcess vp : varList) {
      if (name.equals(vp.v.getFullName()) && (vp.isCoordinateVariable || vp.isCoordinateAxis))
        return vp;
    }
    return null;
  }

  // track coordinate variables
  protected void addCoordinateVariable(Dimension dim, VarProcess vp) {
    List<VarProcess> list = coordVarMap.get(dim);
    if (list == null) {
      list = new ArrayList<>();
      coordVarMap.put(dim, list);
    }
    if (!list.contains(vp))
      list.add(vp);
  }

  /**
   * Wrap each variable in the dataset with a VarProcess object.
   */
  public class VarProcess {
    public NetcdfDataset ds;
    public Variable v;

    // coord axes
    public boolean isCoordinateVariable;
    public boolean isCoordinateAxis;
    public AxisType axisType;
    public String coordAxes, coordSys, coordSysFor, coordVarAlias, positive, coordAxisTypes;
    public String coordinates;// CF : partial coordAxes
    public CoordinateAxis axis; // if its made into a Coordinate Axis, this is not null

    // coord systems
    public boolean isCoordinateSystem;
    public String coordTransforms;
    public CoordinateSystem cs;

    // coord transform
    public boolean isCoordinateTransform;
    public String coordTransformType;
    public CoordinateTransform ct;

    /**
     * Wrap the given variable.
     * Identify Coordinate Variables.
     * Process all _Coordinate attributes.
     *
     * @param ds dataset container
     * @param v  wrap this Variable
     */
    private VarProcess(NetcdfDataset ds, Variable v) {
      this.ds = ds;
      this.v = v;
      VariableEnhanced ve = (VariableEnhanced) v;
      isCoordinateVariable = v.isCoordinateVariable() || null != v.findAttribute(_Coordinate.AliasForDimension);
      if (isCoordinateVariable) {
        v.isCoordinateVariable(); // DEBUG
        addCoordinateVariable(v.getDimension(0), this);
        parseInfo.format(" Coordinate Variable added = %s for dimension %s%n", v.getFullName(), v.getDimension(0));
      }

      Attribute att = v.findAttributeIgnoreCase(_Coordinate.AxisType);
      if (att != null) {
        String axisName = att.getStringValue();
        axisType = AxisType.getType(axisName);
        isCoordinateAxis = true;
        parseInfo.format(" Coordinate Axis added = %s type= %s%n", v.getFullName(), axisName);
      }

      coordVarAlias = ds.findAttValueIgnoreCase(v, _Coordinate.AliasForDimension, null);
      if (coordVarAlias != null) {
        coordVarAlias = coordVarAlias.trim();
        if (v.getRank() != 1) {
          parseInfo.format("**ERROR Coordinate Variable Alias %s has rank %d%n", v.getFullName(), v.getRank());
          userAdvice.format("**ERROR Coordinate Variable Alias %s has rank %d%n", v.getFullName(), v.getRank());
        } else {
          Dimension coordDim = v.getGroup().findDimension(coordVarAlias);
          if (coordDim != null) {
            Dimension vDim = v.getDimension(0);
            if (!coordDim.equals(vDim)) {
              parseInfo.format("**ERROR Coordinate Variable Alias %s names wrong dimension %s%n", v.getFullName(), coordVarAlias);
              userAdvice.format("**ERROR Coordinate Variable Alias %s names wrong dimension %s%n", v.getFullName(), coordVarAlias);
            } else {
              isCoordinateAxis = true;
              addCoordinateVariable(coordDim, this);
              parseInfo.format(" Coordinate Variable Alias added = %s for dimension= %s%n", v.getFullName(), coordVarAlias);
            }
          }
        }
      }

      positive = ds.findAttValueIgnoreCase(v, _Coordinate.ZisPositive, null);
      if (positive == null)
        positive = ds.findAttValueIgnoreCase(v, CF.POSITIVE, null);
      else {
        isCoordinateAxis = true;
        positive = positive.trim();
        parseInfo.format(" Coordinate Axis added(from positive attribute ) = %s for dimension= %s%n", v.getFullName(), coordVarAlias);
      }

      coordAxes = ds.findAttValueIgnoreCase(v, _Coordinate.Axes, null);
      coordSys = ds.findAttValueIgnoreCase(v, _Coordinate.Systems, null);
      coordSysFor = ds.findAttValueIgnoreCase(v, _Coordinate.SystemFor, null);
      coordTransforms = ds.findAttValueIgnoreCase(v, _Coordinate.Transforms, null);
      isCoordinateSystem = (coordTransforms != null) || (coordSysFor != null);

      coordAxisTypes = ds.findAttValueIgnoreCase(v, _Coordinate.AxisTypes, null);
      coordTransformType = ds.findAttValueIgnoreCase(v, _Coordinate.TransformType, null);
      isCoordinateTransform = (coordTransformType != null) || (coordAxisTypes != null);
    }

    public boolean isData() {
      return !isCoordinateVariable && !isCoordinateAxis && !isCoordinateSystem && !isCoordinateTransform;
    }

    public boolean maybeData() {
      return !isCoordinateVariable && !isCoordinateSystem && !isCoordinateTransform;
    }

    public boolean hasCoordinateSystem() {
      return ((VariableEnhanced) v).getCoordinateSystems().size() > 0;
    }

    public String toString() {
      return v.getShortName();
    }

    /**
     * Turn the variable into a coordinate axis, if not already. Add to the dataset, replacing variable if needed.
     *
     * @return variable as a coordinate axis
     */
    public CoordinateAxis makeIntoCoordinateAxis() {
      if (axis != null)
        return axis;

      // if not a CoordinateAxis, will turn into one
      v = axis = ds.addCoordinateAxis((VariableDS) v);

      if (axisType != null) {
        axis.setAxisType(axisType);
        axis.addAttribute(new Attribute(_Coordinate.AxisType, axisType.toString()));

        if (((axisType == AxisType.Height) || (axisType == AxisType.Pressure) || (axisType == AxisType.GeoZ)) &&
                (positive != null)) {
          axis.setPositive(positive);
          axis.addAttribute(new Attribute(_Coordinate.ZisPositive, positive));
        }
      }
      return axis;
    }

    /**
     * Create a Coordinate System object, using the list of coordinate axis names in the
     * (required) axes field.
     */
    public void makeCoordinateSystem() {

      // find referenced coordinate axes
      List<CoordinateAxis> axesList = new ArrayList<>();
      if (coordAxes != null) {
        StringTokenizer stoker = new StringTokenizer(coordAxes); // _CoordinateAxes attribute
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, this);
          if (ap != null) {
            CoordinateAxis axis = ap.makeIntoCoordinateAxis();
            if (!axesList.contains(axis)) axesList.add(axis);
          } else {
            parseInfo.format(" Cant find axes %s for Coordinate System %s%n", vname, v.getFullName());
            userAdvice.format(" Cant find axes %s for Coordinate System %s%n", vname, v.getFullName());
          }
        }
      }

      if (axesList.size() == 0) {
        parseInfo.format(" No axes found for Coordinate System %s%n", v.getFullName());
        userAdvice.format(" No axes found for Coordinate System %s%n", v.getFullName());
        return;
      }

      // create the coordinate system
      cs = new CoordinateSystem(ds, axesList, null);
      ds.addCoordinateSystem(cs);

      parseInfo.format(" Made Coordinate System %s for ", cs.getName());
      v.getNameAndDimensions(parseInfo, true, false);
      parseInfo.format(" from %s%n", coordAxes);
    }

    /**
     * Create a list of coordinate axes for this data variable.
     * Use the list of names in axes or coordinates field.
     *
     * @param addCoordVariables if true, add any coordinate variables that are missing.
     * @return list of coordinate axes for this data variable.
     */
    public List<CoordinateAxis> findCoordinateAxes(boolean addCoordVariables) {
      List<CoordinateAxis> axesList = new ArrayList<>();

      if (coordAxes != null) { // explicit axes
        StringTokenizer stoker = new StringTokenizer(coordAxes);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, this);
          if (ap != null) {
            CoordinateAxis axis = ap.makeIntoCoordinateAxis();
            if (!axesList.contains(axis)) axesList.add(axis);
          }
        }
      } else if (coordinates != null) { // CF partial listing of axes
        StringTokenizer stoker = new StringTokenizer(coordinates);
        while (stoker.hasMoreTokens()) {
          String vname = stoker.nextToken();
          VarProcess ap = findVarProcess(vname, this);
          if (ap != null) {
            CoordinateAxis axis = ap.makeIntoCoordinateAxis(); // LOOK check if its legal
            if (!axesList.contains(axis)) axesList.add(axis);
          }
        }
      }

      if (addCoordVariables) {
        for (Dimension d : v.getDimensions()) {
          List<VarProcess> coordVars = coordVarMap.get(d);
          if (coordVars == null) continue;
          for (VarProcess vp : coordVars) {
            CoordinateAxis axis = vp.makeIntoCoordinateAxis();
            if (!axesList.contains(axis)) axesList.add(axis);
          }
        }
      }

      return axesList;
    }

    void addCoordinateTransform(CoordinateTransform ct) {
      if (cs == null) {
        parseInfo.format("  %s: no CoordinateSystem for CoordinateTransformVariable: %s%n", v.getFullName(), ct.getName());
        return;
      }
      cs.addCoordinateTransform(ct);
    }

  }

  protected VariableDS makeCoordinateTransformVariable(NetcdfDataset ds, CoordinateTransform ct) {
    VariableDS v = CoordTransBuilder.makeDummyTransformVariable(ds, ct);
    parseInfo.format("  made CoordinateTransformVariable: %s%n", ct.getName());
    return v;
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.catalog;

import ucar.nc2.units.DateType;

import java.util.*;
import java.net.*;

/**
 * Concrete implementation of a Thredds catalog object. Use this when you are constructing or modifying.
 *
 * @author john caron
 * @see InvCatalog
 */

public class InvCatalogImpl extends InvCatalog {
  private String createFrom;
  private List<DataRootConfig> roots = new ArrayList<DataRootConfig>();

  // validation
  private StringBuilder log = new StringBuilder();
  private boolean hasError = false;
  private boolean isStatic = false;

  /**
   * Munge this catalog so the given dataset is the top catalog.
   *
   * @param ds make this top; must be existing dataset in this catalog.
   *
   * @deprecated in favor of thredds.catalog.util.DeepCopyUtils.subsetCatalogOnDataset
   */
  public void subset(InvDataset ds) {
    InvDatasetImpl dataset = (InvDatasetImpl) ds;

    // Make all inherited metadata local.
    dataset.transferMetadata(dataset, true);

    topDataset = dataset;
    datasets.clear(); // throw away the rest
    datasets.add(topDataset);

    // parent lookups need to be local
    //InvService service = dataset.getServiceDefault();
    //if (service != null) LOOK
    //  dataset.serviceName = service.getName();
    dataset.dataType = dataset.getDataType();

    // all properties need to be local
    // LOOK dataset.setPropertiesLocal( new ArrayList(dataset.getProperties()));

    // next part requires this before it
    dataset.setCatalog(this);
    dataset.parent = null;

    // any referenced services need to be local
    List<InvService> services = new ArrayList<InvService>(dataset.getServicesLocal());
    findServices(services, dataset);
    dataset.setServicesLocal(services);

    finish();
  }

//  /**
//   * Return a catalog with a shallow copy of the given dataset as the top dataset.
//   *
//   * @param dataset the dataset to copy into a new catalog
//   * @return A new catalog with a shallow copy of the given dataset as the top dataset.
//   */
//  public InvCatalogImpl subsetShallowCopy( InvDatasetImpl dataset)
//  {
//    InvCatalogImpl newCatalog = new InvCatalogImpl( null, "1.0.2", null);
//    InvDatasetImpl newTopDataset = new InvDatasetImpl( null, dataset.getName());
//    newTopDataset.transferMetadata( dataset );
//    if (dataset.getAccess() != null && ! dataset.getAccess().isEmpty())
//    {
//      newTopDataset.addAccess( dataset.getAccess() );
//    }
//    dataset.getDatasets();
//    dataset.getID();
//    dataset.getServiceName();
//    dataset.getServicesLocal();
//    dataset.getUrlPath();
//
//    // ToDo LOOK Probably fine for throw away catalog that is only generated so that XML can be generated.
//    dataset.finish();
//
//    newCatalog.addDataset( newTopDataset );
//
//    return newCatalog;
//  }

  void findServices(List<InvService> result, InvDataset ds) {
    if (ds instanceof InvCatalogRef) return;

    // look for access elements with unresolved services
    for (InvAccess a : ds.getAccess()) {
      InvService s = a.getService();
      InvDataset d = a.getDataset();
      if (null == d.findService(s.getName()) && !(result.contains(s)))
        result.add(s);
    }

    // recurse into nested datasets
    for (InvDataset nested : ds.getDatasets()) {
      findServices(result, nested);
    }
  }

  /**
   * Munge this catalog to remove any dataset that doesnt pass through the filter.
   *
   * @param filter remove datasets that dont pass this filter.
   */
  public void filter(DatasetFilter filter) {
    mark(filter, topDataset);
    delete(topDataset);
    this.filter = filter;
  }

  // got to keep it for catalogRef's
  private DatasetFilter filter = null;

  protected DatasetFilter getDatasetFilter() {
    return filter;
  }

  // see if this ds should be filtered out.
  // if so, setMark and return true.
  // if any nested dataset are kept, then keep it.
  // unread CatalogRefs are always kept.
  private boolean mark(DatasetFilter filter, InvDatasetImpl ds) {
    if (ds instanceof InvCatalogRef) {
      InvCatalogRef catRef = (InvCatalogRef) ds;
      if (!catRef.isRead()) return false;
    }

    // recurse into nested datasets first
    boolean allMarked = true;
    for (InvDataset nested : ds.getDatasets()) {
      allMarked &= mark(filter, (InvDatasetImpl) nested);
    }
    if (!allMarked) return false;

    if (filter.accept(ds) >= 0)
      return false;

    // mark for deletion
    ds.setMark(true);
    if (debugFilter) System.out.println(" mark " + ds.getName());
    return true;
  }

  private boolean debugFilter = false;

  // remove marked datasets
  private void delete(InvDatasetImpl ds) {
    if (ds instanceof InvCatalogRef) {
      InvCatalogRef catRef = (InvCatalogRef) ds;
      if (!catRef.isRead()) return;
    }

    Iterator iter = ds.getDatasets().iterator();
    while (iter.hasNext()) {
      InvDatasetImpl nested = (InvDatasetImpl) iter.next();
      if (nested.getMark()) {
        iter.remove();
        if (debugFilter) System.out.println(" remove " + nested.getName());
      } else
        delete(nested);
    }
  }

  ///////////////////////////////////////////////////////////////////////////////
  // construction and internals

  /**
   * Construct an InvCatalog.
   * You must call finish() after all objects are added.
   *
   * @param name    : catalog name.
   * @param version : catalog version.
   * @param baseURI : catalog base URI (external).
   */
  public InvCatalogImpl(String name, String version, URI baseURI) {
    this(name, version, null, baseURI);
  }

  /**
   * Construct an InvCatalog.
   * You must call finish() after all objects are added.
   *
   * @param name    : catalog name.
   * @param version : catalog version.
   * @param expires : date/time catalog expires.
   * @param baseURI : catalog base URI  (external).
   */
  public InvCatalogImpl(String name, String version, DateType expires, URI baseURI) {
    this.name = name;
    this.version = version;
    this.expires = expires;
    this.baseURI = baseURI;
  }

  /**
   * Finish constructing after all elements have been added or modified.
   * This routine will do any needed internal consistency work.
   * Its ok to call multiple times.
   *
   * @return true if successful.
   */
  public boolean finish() {

    // make topDataset if needed
    //if (topDataset == null) {

    if (datasets.size() == 1) { // already only one; make it top
      topDataset = (InvDatasetImpl) datasets.get(0);

    } else { // create one
      topDataset = new InvDatasetImpl(null, name == null ? "Top Dataset" : name);
      for (InvDataset dataset : datasets)
        topDataset.addDataset((InvDatasetImpl) dataset);
      topDataset.setServicesLocal(services);
    }
    //}
    topDataset.setCatalog(this);

    // build dataset hash table
    dsHash = new HashMap<String, InvDataset>();
    addDatasetIds(topDataset);

    // recurse through the datasets and finish them
    return topDataset.finish();
  }

  private void addDatasetIds(InvDatasetImpl ds) {
    addDatasetByID(ds);

    if (ds instanceof InvCatalogRef) return;
    //if (ds instanceof InvDatasetFmrc) return;

    // recurse into nested
    for (InvDataset invDataset : ds.getDatasets()) {
      InvDatasetImpl nested = (InvDatasetImpl) invDataset;
      addDatasetIds(nested);
    }
  }

  /**
   * Add Dataset to internal hash.
   *
   * @param ds : add this dataset if ds.getID() != null
   * @see InvCatalog#findDatasetByID
   */
  public void addDatasetByID(InvDatasetImpl ds) {
    //if (ds.getID() != null && ds.getID().startsWith("null"))
    //  System.out.printf("HEY addDatasetByID %s%n", ds.getID());

    if (ds.getID() != null)
      dsHash.put(ds.getID(), ds);
  }

  /**
   * Find the dataset in this catalog by its ID. If found, remove it.
   *
   * @param ds Remove this dataset from the hash
   */
  public void removeDatasetByID(InvDatasetImpl ds) {
    if (ds.getID() != null)
      dsHash.remove(ds.getID());
  }

  /**
   * Add Dataset (1.0)
   *
   * @param ds add this dataset
   */
  public void addDataset(InvDatasetImpl ds) {
    if (ds != null)
      datasets.add(ds);
  }

  /**
   * Remove the given dataset from this catalog if it is a direct child of this catalog.
   *
   * @param ds remove this dataset
   * @return true if found and removed
   */
  public boolean removeDataset(InvDatasetImpl ds) {
    if (this.datasets.remove(ds)) {
      ds.setParent(null);
      removeDatasetByID(ds);
      return true;
    }
    return false;
  }

  /**
   * Replace the given dataset if it is a nested dataset.
   *
   * @param remove - the dataset element to be removed
   * @param add    - the dataset element to be added
   * @return true on success
   */
  public boolean replaceDataset(InvDatasetImpl remove, InvDatasetImpl add) {
    if (topDataset.equals(remove)) {
      topDataset = add;
      topDataset.setCatalog(this);
    }
    for (int i = 0; i < datasets.size(); i++) {
      InvDataset dataset = datasets.get(i);
      if (dataset.equals(remove)) {
        datasets.set(i, add);
        removeDatasetByID(remove);
        addDatasetByID(add);
        return true;
      }
    }
    return false;
  }

  /**
   * Add Property (1.0)
   *
   * @param p add this property
   */
  public void addProperty(InvProperty p) {
    properties.add(p);
  }

  /**
   * Add Service (1.0)
   *
   * @param s add this service
   */
  public void addService(InvService s) {
    if (s == null)
      throw new IllegalArgumentException("Service to add was null.");

    // While adding a service, there are three possible results:
    if (s.getName() != null) {
      Object obj = serviceHash.get(s.getName());
      if (obj == null) {
        // 1) No service with matching name entry was found, add given service;
        serviceHash.put(s.getName(), s);
        services.add(s);
        return;

      } else {
        // A service with matching name was found.
        if (s.equals(obj)) {
          // 2) matching name entry, objects are equal so OK;
          return;
        } else {
          // 3) matching name entry, objects are not equal so ???
          // @todo throw an exception???
          // Currently just dropping given service
          log.append("Multiple Services with the same name\n");
          return;
        }
      }
    }
  }

  /**
   * Add top-level InvDataset to this catalog.
   *
   * @deprecated Use addDataset() instead; datamodel now allows multiple top level datasets.
   */
  public void setDataset(InvDatasetImpl ds) {
    topDataset = ds;
    addDataset(ds);
  }

  /**
   * String describing how the catalog was created, for debugging.
   *
   * @return how the catalog was created, for debugging
   */
  public String getCreateFrom() {
    return createFrom;
  }

  /**
   * Set how the catalog was created, for debugging.
   * @param createFrom how the catalog was created, for debugging
   */
  public void setCreateFrom(String createFrom) {
    this.createFrom = createFrom;
  }

  /**
   * Set the catalog base URI.
   * Its used to resolve reletive URLS.
   * @param baseURI set to this
   */
  public void setBaseURI(URI baseURI) {
    this.baseURI = baseURI;
  }

  /**
   * @return the catalog base URI.
   */
  public URI getBaseURI() {
    return baseURI;
  }

  /*
   * @return DTD string
   *
  public String getDTDid() {
    return dtdID;
  }

  /*
   * set DTD
   *
  public void setDTDid(String dtdID) {
    this.dtdID = dtdID;
  } */

  /**
   * Set the expires date after which the catalog is no longer valid.
   *
   * @param expiresDate a {@link DateType} representing the date after which the catlog is no longer valid.
   */
  public void setExpires(DateType expiresDate) {
    this.expires = expiresDate;
  }

  /**
   * Check if there is a fatal error and catalog should not be used.
   *
   * @return true if catalog not useable.
   */
  public boolean hasFatalError() {
    return hasError;
  }

  /**
   * Append an error message to the message log. Call check() to get the log when
   * everything is done.
   *
   * @param message   append this message to log
   * @param isInvalid true if this is a fatal error.
   */
  public void appendErrorMessage(String message, boolean isInvalid) {
    log.append(message);
    hasError = hasError | isInvalid;
  }

  /**
   * Check internal data structures.
   *
   * @param out  : print errors here
   * @param show : print messages for each object (debug)
   * @return true if no fatal consistency errors.
   */
  public boolean check(StringBuilder out, boolean show) {
    boolean isValid = !hasError;
    out.append("----Catalog Validation\n");

    if (log.length() > 0)
      out.append(log);

    if (show) System.out.println(" catalog valid = " + isValid);

    //if (topDataset != null)
    //  isValid &= topDataset.check( out, show);

    for (InvDataset ds : datasets) {
      InvDatasetImpl dsi = (InvDatasetImpl) ds;
      dsi.check(out, show); // cant make it invalid !!
    }

    return isValid;
  }

  public String getLog() {
    return log.toString();
  }

  /**
   * Debugging: dump entire data structure.
   *
   * @return String representation.
   */
  public String dump() {
    StringBuilder buff = new StringBuilder(1000);
    buff.setLength(0);

    buff.append("Catalog <").append(getName())
        .append("> <").append(getVersion())
        .append("> <").append(getCreateFrom()).append(">\n");
    buff.append(topDataset.dump(2));

    return buff.toString();
  }

  /*
   * Add a PropertyChangeEvent Listener. THIS IS EXPERIMENTAL DO NOT RELY ON.
   * Throws a PropertyChangeEvent:
   *

propertyName = "InvCatalogRefInit", getNewValue() = InvCatalogRef that was just initialized
   *

   * @param l the listener
   *
  public void addPropertyChangeListener(PropertyChangeListener l) {
    if (listenerList == null) listenerList = new EventListenerList();
    listenerList.add(PropertyChangeListener.class, l);
  }

  /**
   * Remove a PropertyChangeEvent Listener.
   * @param l the listener
   *
  public void removePropertyChangeListener(PropertyChangeListener l) {
    listenerList.remove(PropertyChangeListener.class, l);
  }

  private EventListenerList listenerList = null;

  // PropertyChangeEvent(Object source, String propertyName, Object oldValue, Object newValue)
  void firePropertyChangeEvent(PropertyChangeEvent event) {
    // System.out.println("firePropertyChangeEvent "+event);
    if (listenerList == null) return;

    // Process the listeners last to first
    Object[] listeners = listenerList.getListenerList();
    for (int i = listeners.length - 2; i >= 0; i -= 2) {
      if (listeners[i] == PropertyChangeListener.class) {
        ((PropertyChangeListener) listeners[i + 1]).propertyChange(event);
      }
    }
  } */

  /**
   * This finds the topmost catalog, even when its a InvCatalogRef.
   * Used to throw a PropertyChange event on the top catalog.
   *
   * @return top catalog
   */
  InvCatalogImpl getTopCatalog() {
    return (top == null) ? this : top;
  }

  void setTopCatalog(InvCatalogImpl top) {
    this.top = top;
  }

  private InvCatalogImpl top = null;

  ////////////////////////////////////////////
  /* private InvCatalogFactory factory = null;
  // private InvCatalogConvertIF converter = null;

  // this is how catalogRefs read their catalogs
  InvCatalogFactory getCatalogFactory() {
    return factory;
  }

  void setCatalogFactory(InvCatalogFactory factory) {
    this.factory = factory;
  }

  // track converter
  InvCatalogConvertIF getCatalogConverter() {
    return converter;
  }

  void setCatalogConverter(InvCatalogConvertIF converter) {
    this.converter = converter;
  } */

  /*
   * Set the connverter to 1.0, typically to write a 0.6 out to a 1.0
   *
  public void setCatalogConverterToVersion1() {
    setCatalogConverter(factory.getCatalogConverter(XMLEntityResolver.CATALOG_NAMESPACE_10));
  } */

  /**
   * Write the catalog as an XML document to the specified stream.
   *
   * @param os write to this OutputStream
   * @throws java.io.IOException on an error.
   */
  public void writeXML(java.io.OutputStream os) throws java.io.IOException {
    InvCatalogConvertIF converter = InvCatalogFactory.getDefaultConverter();
    converter.writeXML(this, os);
  }

  /**
   * Write the catalog as an XML document to the specified stream.
   *
   * @param os  write to this OutputStream
   * @param raw if true, write original (server) version, else write client version
   * @throws java.io.IOException on an error.
   */
  public void writeXML(java.io.OutputStream os, boolean raw) throws java.io.IOException {
    InvCatalogConvertIF converter = InvCatalogFactory.getDefaultConverter();
    converter.writeXML(this, os, raw);
  }

  //////////////////////////////////////////////////////////////////////////
  /**
   * Get dataset roots.
   *
   * @return List of InvProperty. May be empty, may not be null.
   */
  public java.util.List<DataRootConfig> getDatasetRoots() {
    return roots;
  }

  /**
   * Add Dataset Root, key = path,  value = location.
   * @param root add a dataset root
   */
  public void addDatasetRoot(DataRootConfig root) {
    roots.add(root);
  }

  /**
   * InvCatalogImpl elements with same values are equal.
   */
  public boolean equals(Object o) {
    if (this == o) return true;
    if (!(o instanceof InvCatalogImpl)) return false;
    return o.hashCode() == this.hashCode();
  }

  /**
   * Override Object.hashCode() to implement equals.
   */
  public int hashCode() {
    if (hashCode == 0) {
      int result = 17;
      if (null != getName())
        result = 37 * result + getName().hashCode();
      result = 37 * result + getServices().hashCode();
      result = 37 * result + getDatasets().hashCode();
      hashCode = result;
    }
    return hashCode;
  }

  private volatile int hashCode = 0; // Bloch, item 8

  public boolean isStatic() {
    return isStatic;
  }

  public void setStatic(boolean aStatic) {
    isStatic = aStatic;
  }
}


/* This file is Copyright (c) 2005 Robert Alten Simons (info@cohort.com).
 * See the MIT/X-like license in LICENSE.txt.
 * For more information visit www.cohort.com or contact info@cohort.com.
 */
package ucar.nc2.ogc.erddap.util;

import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.TimeZone;

/**
 * This class has static methods for dealing with dates and times.
 *
 *
newGCalendar only accounts for daylight savings if
 * your computer is correctly
 * set up. E.g., in Windows, make sure "Start : Control Panel :
 * Date and Time : Time Zone : Automatically adjust clock for daylight
 * savings changes" is checked. Otherwise, the TimeZone used by GregorianCalendar
 * will be for standard time (not including daylight savings time, if any).
 *
 *
Comments about working with Java's GregorianCalendar class:
 *

 *
GregorianCalendar holds millis since Jan 1, 1970 and a timeZone
 *   which influences the values that get/set deal with.
 *
Using a simpleDateFormat to parse a string to a Gregorian Calendar:
 *   the simpleDateFormat has a timeZone which specified where the
 *   the strings value is from (e.g., 2005-10-31T15:12:10 in PST).
 *   When parsed, it is then interpreted by the GregorianCalendar's timeZone
 *   (e.g., it was 3pm PST but now I'll treat it as 6pm EST).
 *
Similarly, using a simpleDateFormat to format a Gregorian Calendar
 *   to a String:
 *   the simpleDateFormat has a timeZone which specified where the
 *   the strings value will be for (e.g., 6pm EST will be formatted as
 *   5pm Central).
 *

 *
 *
But this class seeks to simplify things to the more common cases
 * of parsing and formatting using the same time zone as the GregorianCalendar
 * class, and offering GregorianCalendar constructors for Local (with
 * daylight savings if that is what your area does) and Zulu (aka GMT and UTC,
 * which doesn't ever use daylight savings).
 *
 *
A summary of ISO 8601 Date Time formats is at
 * http://www.cl.cam.ac.uk/~mgk25/iso-time.html
 * http://en.wikipedia.org/wiki/ISO_8601
 * and http://dotat.at/tmp/ISO_8601-2004_E.pdf
 * (was http://www.iso.org/iso/date_and_time_format)
 * and years B.C at http://www.tondering.dk/claus/cal/node4.html#SECTION00450000000000000000
 *
 *
Calendar2 does not use ERA designations. It uses negative year values for B.C years
 * (calendar2Year = 1 - BCYear).  Note that BCYears are 1..., so 1 BC is calendar2Year 0 (or 0000),
 * and 2 BC is calendar2Year -1 (or -0001).
 *
 */
public class ErddapCalendar2 {
    //useful static variables
    public final static int YEAR        = Calendar.YEAR;
    public final static int MILLISECOND = Calendar.MILLISECOND;

    public final static int SECONDS_PER_MINUTE = 60;
    public final static int SECONDS_PER_HOUR   = 60 * 60; //3600
    public final static int SECONDS_PER_DAY    = 24 * 60 * 60; //86400   31Days=2678400  365days=31536000

    public final static TimeZone zuluTimeZone = TimeZone.getTimeZone("Zulu");

    /**
     * This converts a string "[units] since [isoDate]"
     * (e.g., "minutes since 1985-01-01") into
     * a baseSeconds (seconds since 1970-01-01) and a factor ("minutes" returns 60).
     *
So simplistically, epochSeconds = storedTime * factor + baseSeconds.
     *
Or simplistically, storedTime = (epochSeconds - baseSeconds) / factor.
     *
     *
WARNING: don't use the equations above. Use unitsSinceToEpochSeconds or
     * epochSecondsToUnitsSince which correctly handle special cases.
     *
     * @param tsUnits e.g., "minutes since 1985-01-01".
     *   This may include hours, minutes, seconds, decimal, and Z or timezone offset (default=Zulu).
     * @return double[]{baseSeconds, factorToGetSeconds}
     * @throws Exception if trouble (tsUnits is null or invalid)
     */
    public static double[] getTimeBaseAndFactor(String tsUnits) throws Exception {
        String errorInMethod = ErddapString2.ERROR + " in Calendar2.getTimeBaseAndFactor(" + tsUnits + "):\n";

        if (tsUnits == null) {
            throw new NullPointerException(errorInMethod + "tsUnits must be non-null.");
        }

        int sincePo = tsUnits.toLowerCase().indexOf(" since ");
        if (sincePo <= 0)
            throw new IllegalArgumentException(errorInMethod + "units string doesn't contain \" since \".");
        double factorToGetSeconds = factorToGetSeconds(tsUnits.substring(0, sincePo));
        GregorianCalendar baseGC = parseISODateTimeZulu(tsUnits.substring(sincePo + 7));
        double baseSeconds = baseGC.getTimeInMillis() / 1000.0;

        return new double[]{baseSeconds, factorToGetSeconds};
    }

    /**
     * This returns the factor to multiply by 'units' data to get seconds
     * data (e.g., "minutes" returns 60).
     * This is used for part of dealing with udunits-style "minutes since 1970-01-01"-style
     * strings.
     *
     * @param units
     * @return the factor to multiply by 'units' data to get seconds data.
     *    Since there is no exact value for months or years, this returns
     *    special values of 30*SECONDS_PER_DAY and 360*SECONDS_PER_DAY, respectively.
     * @throws Exception if trouble (e.g., units is null or not an expected value)
     */
    public static double factorToGetSeconds(String units) throws Exception {
        units = units.trim().toLowerCase();
        if (units.equals("ms") ||
            units.equals("msec") ||
            units.equals("msecs") ||
            units.equals("millis") ||
            units.equals("millisec") ||
            units.equals("millisecs") ||
            units.equals("millisecond") ||
            units.equals("milliseconds")) return 0.001;
        if (units.equals("s") ||
            units.equals("sec") ||
            units.equals("secs") ||
            units.equals("second") ||
            units.equals("seconds")) return 1;
        if (units.equals("m") ||
            units.equals("min") ||
            units.equals("mins") ||
            units.equals("minute") ||
            units.equals("minutes")) return SECONDS_PER_MINUTE;
        if (units.equals("h") ||
            units.equals("hr") ||
            units.equals("hrs") ||
            units.equals("hour") ||
            units.equals("hours")) return SECONDS_PER_HOUR;
        if (units.equals("d") ||
            units.equals("day") ||
            units.equals("days")) return SECONDS_PER_DAY;
        if (units.equals("week") ||
            units.equals("weeks")) return 7 * SECONDS_PER_DAY;
        if (units.equals("mon") ||
            units.equals("mons") ||
            units.equals("month") ||
            units.equals("months")) return 30 * SECONDS_PER_DAY;
        if (units.equals("yr") ||
            units.equals("yrs") ||
            units.equals("year") ||
            units.equals("years")) return 360 * SECONDS_PER_DAY;

        throw new RuntimeException(
                ErddapString2.ERROR + " in Calendar2.factorToGetSeconds: units=\"" + units + "\" is invalid.");
    }

    /**
     * Get a GregorianCalendar object with the current UTC
     * (A.K.A., GMT or Zulu) time and a UTC time zone.
     * You can find the current Zulu/GMT time at: http://www.xav.com/time.cgi
     * Info about UTC vs GMT vs TAI... see http://www.leapsecond.com/java/gpsclock.htm.
     * And there was another good site... can't find it.
     *
     * @return the GregorianCalendar object for right now (Zulu time zone)
     */
    public static GregorianCalendar newGCalendarZulu() {
        return new GregorianCalendar(zuluTimeZone);
    }

    /** This parses n int values from s and stores results in resultsN (or leaves
     * items in resultsN untouched if no value available).
     *
     * @param s the date time string
     * @param separatorN is the separators (use "\u0000" to match any non-digit).
     *    (± matches + or - and that becomes part of the number)
     *    (. matches . or , (the European decimal point))
     * @param resultsN should initially have the defaults and
     *   will receive the results.  If trouble, resultsN[0] will be Integer.MAX_VALUE,
     *   so caller can throw exception with good error message.
     */
    private static void parseN(String s, char separatorN[], int resultsN[]) {
        //ensure s starts with a digit
        if (s == null)
            s = "";
        s = s.trim();
        int sLength = s.length();
        if (sLength < 1 || !(s.charAt(0) == '-' || ErddapString2.isDigit(s.charAt(0)))) {
            resultsN[0] = Integer.MAX_VALUE;
            return;
        }
        int po1, po2 = -1;
        //String2.log("parseN " + s);

        //search for digits, non-digit.   "1970-01-01T00:00:00.000-01:00"
        boolean mMode = s.charAt(0) == '-'; //initial '-' is required and included when evaluating number
        int nParts = separatorN.length;
        for (int part = 0; part < nParts; part++) {
            if (po2 + 1 < sLength) {
                //accumulate digits
                po1 = po2 + 1;
                po2 = po1;
                if (mMode) {
                    if (po2 < sLength && s.charAt(po2) == '-') po2++;
                    else {resultsN[0] = Integer.MAX_VALUE; return; }
                }
                while (po2 < sLength && ErddapString2.isDigit(s.charAt(po2))) po2++; //digit

                //if no number, return; we're done
                if (po2 == po1)
                    return;
                if (part > 0 && separatorN[part - 1] == '.') {
                    resultsN[part] = ErddapMath2.roundToInt(1000 *
                            ErddapString2.parseDouble("0." + s.substring(po1, po2)));
                    //String2.log("  millis=" + resultsN[part]);
                } else {
                    resultsN[part] = ErddapString2.parseInt(s.substring(po1, po2));
                }

                //if invalid number, return trouble
                if (resultsN[part] == Integer.MAX_VALUE) {
                    resultsN[0] = Integer.MAX_VALUE;
                    return;
                }

                //if no more source characters, we're done
                if (po2 >= sLength) {
                    //String2.log("  " + String2.toCSSVString(resultsN));
                    return;
                }

                //if invalid separator, stop trying to read more; return trouble
                mMode = false;
                char ch = s.charAt(po2);
                if (ch == ',')
                    ch = '.';
                if (separatorN[part] == '\u0000') {

                } else if (separatorN[part] == '±') {
                    if (ch == '+') { //do nothing
                    }else if (ch == '-') {
                        po2--; //number starts with -
                        mMode = true;
                    } else {
                        resultsN[0] = Integer.MAX_VALUE; return;
                    }
                } else if (ch != separatorN[part]) { //if not exact match ...
                    //if current part is ':' or '.' and not matched, try to skip forward to '±'
                    if ((separatorN[part] == ':' || separatorN[part] == '.') &&
                        part < nParts - 1) {
                        int pmPart = ErddapString2.indexOf(separatorN, '±', part + 1);
                        if (pmPart >= 0) {
                            //String2.log("  jump to +/-");
                            part = pmPart;
                            if (ch == '+') { //do nothing
                            }else if (ch == '-') {
                                po2--; //number starts with -
                                mMode = true;
                            } else {
                                resultsN[0] = Integer.MAX_VALUE; return;
                            }
                            continue;
                        } //if < 0, fall through to failure
                    }
                    resultsN[0] = Integer.MAX_VALUE;
                    return;
                }
            }
        }
    }

    /**
     * This converts an ISO date time string ([-]YYYY-MM-DDTHH:MM:SS.SSS±ZZ:ZZ) into
     *   a GregorianCalendar object.
     *
It is lenient; so Jan 32 is converted to Feb 1;
     *
The 'T' may be any non-digit.
     *
The time zone can be omitted.
     *
The parts at the end of the time can be omitted.
     *
If there is no time, the end parts of the date can be omitted.  Year is required.
     *
This tries hard to be tolerant of non-valid formats (e.g., "1971-1-2", "1971-01")
     *
As of 11/9/2006, NO LONGER TRUE: If year is 0..49, it is assumed to be 2000..2049.
     *
As of 11/9/2006, NO LONGER TRUE: If year is 50..99, it is assumed to be 1950..1999.
     *
If the string is too short, the end of "1970-01-01T00:00:00.000Z" will be added (effectively).
     *
If the string is too long, the excess will be ignored.
     *
If a required separator is incorrect, it is an error.
     *
If the date is improperly formatted, it returns null.
     *
Timezone "Z" or "" is treated as "-00:00" (UTC/Zulu time)
     *
Timezones: e.g., 2007-01-02T03:04:05-01:00 is same as 2007-01-02T04:04:05
     *
     * @param gc a GregorianCalendar object. The dateTime will be interpreted
     *   as being in gc's time zone.
     *   Timezone info is relative to the gc's time zone.
     * @param s the dateTimeString in the ISO format (YYYY-MM-DDTHH:MM:SS.SSS±ZZ:ZZ
     *   or -YYYY-MM-DDTHH:MM:SS.SSS±ZZ:ZZ for years B.C.)
     *   For years B.C., use calendar2Year = 1 - BCYear.
     *   Note that BCYears are 1..., so 1 BC is calendar2Year 0 (or 0000),
     *   and 2 BC is calendar2Year -1 (or -0001).
     *   This supports SS.SSS and SS,SSS (which ISO 8601 prefers!).
     * @return the same GregorianCalendar object, but with the date info
     * @throws RuntimeException if trouble (e.g., gc is null or s is null or
     *    not at least #)
     */
    public static GregorianCalendar parseISODateTime(GregorianCalendar gc,
        String s) {

        if (s == null)
            s = "";
        boolean negative = s.startsWith("-");
        if (negative)
            s = s.substring(1);
        if (s.length() < 1 || !ErddapString2.isDigit(s.charAt(0)))
            throw new RuntimeException(
                    ErddapString2.ERROR + " in parseISODateTime: for first character of dateTime='" + s + "' isn't a digit!");
        if (gc == null)
            throw new RuntimeException(ErddapString2.ERROR + " in parseISODateTime: gc is null!");

        //default ymdhmsmom     year is the only required value
        int ymdhmsmom[] = {Integer.MAX_VALUE, 1, 1, 0, 0, 0, 0, 0, 0};

        //remove trailing Z or "UTC"
        s = s.trim();
        if (Character.toLowerCase(s.charAt(s.length() - 1)) == 'z')
            s = s.substring(0, s.length() - 1).trim();
        if (s.length() >= 3) {
            String last3 = s.substring(s.length() - 3).toLowerCase();
            if (last3.equals("utc") || last3.equals("gmt"))
                s = s.substring(0, s.length() - 3).trim();
        }

        //if e.g., 1970-01-01 00:00:00 0:00, change ' ' to '+' (first ' '->'+' is irrelevant)
        s = ErddapString2.replaceAll(s, ' ', '+');

        //separators (\u0000=any non-digit)
        char separator[] = {'-','-','\u0000',':',':','.','±', ':', '\u0000'};
        parseN(s, separator, ymdhmsmom);
        if (ymdhmsmom[0] == Integer.MAX_VALUE)
            throw new RuntimeException(
                    ErddapString2.ERROR + " in parseISODateTime: dateTime='" + s + "' has an invalid format!");

        //do time zone adjustment
        //String2.log("#7=" + ymdhmsmom[7] + " #8=" + ymdhmsmom[8]);
        if (ymdhmsmom[7] != 0)
            ymdhmsmom[3] -= ymdhmsmom[7];
        if (ymdhmsmom[8] != 0)
            ymdhmsmom[4] -= ymdhmsmom[7] < 0? -ymdhmsmom[8] : ymdhmsmom[8];

        //set gc      month -1 since gc month is 0..
        gc.set((negative? -1 : 1) * ymdhmsmom[0], ymdhmsmom[1] - 1, ymdhmsmom[2],
            ymdhmsmom[3], ymdhmsmom[4], ymdhmsmom[5]);
        gc.set(MILLISECOND, ymdhmsmom[6]);
        gc.get(YEAR); //force recalculations

        return gc;
    }

    /**
     * This converts an ISO (default *ZULU* time zone) date time string ([-]YYYY-MM-DDTHH:MM:SS±ZZ:ZZ) into
     * a GregorianCalendar object with the Zulu time zone.
     * See parseISODateTime documentation.
     *
     * @param s the dateTimeString in the ISO format ([-]YYYY-MM-DDTHH:MM:SS)
     *   This may include hours, minutes, seconds, decimal, and Z or timezone offset (default=Zulu).
     * @return a GregorianCalendar object
     */
    public static GregorianCalendar parseISODateTimeZulu(String s) {
        return parseISODateTime(newGCalendarZulu(), s);
    }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.bufr.writer;

import ucar.nc2.*;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.iosp.bufr.BufrIosp2;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.ma2.*;

import java.io.IOException;
import java.util.List;
import java.util.ArrayList;

/**
 * Class Description.
 *
 * @author caron
 * @since Jun 21, 2008
 */
public class WriteT41_ncFlat {
  private static boolean debug = true;

  public WriteT41_ncFlat(NetcdfFile bufr, String fileOutName, boolean fill) throws IOException, InvalidRangeException {

    try (NetcdfFileWriter ncfile = NetcdfFileWriter.createNew(fileOutName, fill)) {
      if (debug) {
        System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
      }

      // global attributes
      List<Attribute> glist = bufr.getGlobalAttributes();
      for (Attribute att : glist) {
        String useName = N3iosp.makeValidNetcdfObjectName(att.getShortName());
        Attribute useAtt;
        if (att.isArray())
          useAtt = ncfile.addGroupAttribute(null, new Attribute(useName, att.getValues()));
        else if (att.isString())
          useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
        else
          useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
        if (debug) System.out.println("add gatt= " + useAtt);
      }

      // global dimensions
      Dimension obsDim = null;
      for (Dimension oldD : bufr.getDimensions()) {
        String useName = N3iosp.makeValidNetcdfObjectName(oldD.getShortName());
        boolean isRecord = useName.equals("record");
        Dimension newD = ncfile.addDimension(useName, oldD.getLength());
        if (isRecord) obsDim = newD;
        if (debug) System.out.println("add dim= " + newD);
      }

      // Variables
      Structure recordStruct = (Structure) bufr.findVariable(BufrIosp2.obsRecord);
      for (Variable oldVar : recordStruct.getVariables()) {
        if (oldVar.getDataType() == DataType.SEQUENCE) continue;

        String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
        DataType newType = oldVar.getDataType();

        List<Dimension> newDims = new ArrayList<>();
        newDims.add(obsDim);
        for (Dimension dim : oldVar.getDimensions()) {
          newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
        }

        Variable newVar = ncfile.addVariable(null, varName, newType, newDims);
        if (debug) System.out.println("add var= " + newVar);

        // attributes
        List<Attribute> attList = oldVar.getAttributes();
        for (Attribute att : attList) {
          String useName = N3iosp.makeValidNetcdfObjectName(att.getShortName());
          if (att.isArray())
            newVar.addAttribute(new Attribute(useName, att.getValues()));
          else if (att.isString())
            ncfile.addVariableAttribute(varName, useName, att.getStringValue());
          else
            ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
        }
      }

      int total_seq = countSeq(recordStruct);
      Dimension seqD = ncfile.addDimension(null, "seq", total_seq, true, false);

      for (Variable v : recordStruct.getVariables()) {
        if (v.getDataType() != DataType.SEQUENCE) continue;

        Structure seq = (Structure) v;
        for (Variable seqVar : seq.getVariables()) {
          String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName());
          DataType newType = seqVar.getDataType();

          List<Dimension> newDims = new ArrayList<>();
          newDims.add(seqD);
          for (Dimension dim : seqVar.getDimensions()) {
            newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
          }

          Variable newVar = ncfile.addVariable(null, varName, newType, newDims);
          if (debug) System.out.println("add var= " + newVar);

          // attributes
          List<Attribute> attList = seqVar.getAttributes();
          for (Attribute att : attList) {
            String useName = N3iosp.makeValidNetcdfObjectName(att.getShortName());
            if (att.isArray())
              newVar.addAttribute( new Attribute( useName, att.getValues()));
            else if (att.isString())
              ncfile.addVariableAttribute(varName, useName, att.getStringValue());
            else
              ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
          }
        }
      }

      // create the file
      ncfile.create();
      if (debug)
        System.out.println("File Out= " + ncfile.toString());

      // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

      double total = copyVarData(bufr, ncfile, recordStruct);
      ncfile.flush();
      if (debug) System.out.println("FileWriter done total bytes = " + total);
    }
  }

  private int countSeq(Structure recordStruct) throws IOException {
    int total = 0;
    int count = 0;
    int max = 0;

    try (StructureDataIterator iter = recordStruct.getStructureIterator()) {
      while (iter.hasNext()) {
        StructureData sdata = iter.next();
        ArraySequence seq1 = sdata.getArraySequence("seq1");
        int n = seq1.getStructureDataCount();
        total += n;
        count++;
        max = Math.max(max, n);
      }
    }
    if (count > 0 && max > 0) {
      double avg = (double)total / count;
      int wasted = count * max - total;
      double wp = (double) wasted / (count * max);
      System.out.println(" Max = " + max + " avg = " + avg + " wasted = " + wasted + " %= " + wp);
    } else {
      System.out.println(" T41_ncFlat - countSeq called on empty recordStruct" +
              " max = " + max + " count = " + count);
    }

    return total;
  }

  private double copyVarData(NetcdfFile bufr, NetcdfFileWriter ncfile, Structure recordStruct) throws IOException, InvalidRangeException {
    int nrecs = (int) recordStruct.getSize();
    int sdataSize = recordStruct.getElementSize();

    int seqCount = 0;
    double total = 0;
    double totalRecordBytes = 0;
    for (int count = 0; count < nrecs; count++) {

      StructureData recordData = recordStruct.readStructure(count);
      for (StructureMembers.Member m : recordData.getMembers()) {

        if (m.getDataType() == DataType.SEQUENCE) {
          ArraySequence seq1 = recordData.getArraySequence(m);
          try (StructureDataIterator iter = seq1.getStructureDataIterator()) {
            while (iter.hasNext()) {
              StructureData seqData = iter.next();
              for (StructureMembers.Member seqm : seqData.getMembers()) {
                Array data = seqData.getArray(seqm);
                int[] shape = data.getShape();
                int[] newShape = new int[data.getRank() + 1];
                newShape[0] = 1;
                System.arraycopy(shape, 0, newShape, 1, data.getRank());

                int[] origin = new int[data.getRank() + 1];
                origin[0] = seqCount;

                if (debug && (count == 0) && (seqCount == 0)) System.out.println("write to = " + seqm.getName());
                ncfile.write(seqm.getName(), origin, data.reshape(newShape));
              }
              seqCount++;
            }
          }
        } else {

          Array data = recordData.getArray(m);
          int[] shape = data.getShape();
          int[] newShape = new int[data.getRank() + 1];
          newShape[0] = 1;
          System.arraycopy(shape, 0, newShape, 1, data.getRank());

          int[] origin = new int[data.getRank() + 1];
          origin[0] = count;

          if (debug && (count == 0)) System.out.println("write to = " + m.getName());
          ncfile.write(m.getName(), origin, data.reshape(newShape));
        }
      }
      totalRecordBytes += sdataSize;
    }

    total += totalRecordBytes;
    totalRecordBytes /= 1000 * 1000;
    if (debug) System.out.println("write record var; total = " + totalRecordBytes + " Mbytes # recs=" + nrecs);

    return total;
  }

  /**
   * main.
   */
  public static void main(String args[]) throws Exception {

    //String fileIn = "C:/data/dt2/point/bufr/IUA_CWAO_20060202_12.bufr";
    //String fileIn = "C:/data/bufr/edition3/idd/profiler/PROFILER_3.bufr";
    //String fileIn = "C:/data/bufr/edition3/ecmwf/synop.bufr";
    //String fileIn = "R:/testdata2/bufr/edition3/idd/profiler/PROFILER_1.bufr";
    String fileIn = "D:/mlode/bufr/cat.out";
    NetcdfDataset ncf = NetcdfDataset.openDataset(fileIn);
    System.out.println(ncf.toString());

    /* Structure s = (Structure) ncf.findVariable(obsRecord);
    StructureData sdata = s.readStructure(2);
    PrintWriter pw = new PrintWriter(System.out);
    NCdumpW.printStructureData(pw, sdata);  */
    new WriteT41_ncFlat(ncf, "D:/mlode/bufr/cat2.nc", true);

    //Variable v = ncf.findVariable("recordIndex");
    //NCdumpW.printArray(v.read(), "recordIndex", pw, null);

    /* ucar.nc2.Variable v;

    v = ncf.findVariable("trajectory_id");
    if (v != null) {
      Array data = v.read();
      NCdump.printArray(data, v.getName(), System.out, null);
    }
    v = ncf.findVariable("station_id");
    if (v != null) {
      Array data = v.read();
      NCdump.printArray(data, v.getName(), System.out, null);
    }
    v = ncf.findVariable("firstChild");
    if (v != null) {
      Array data = v.read();
      NCdump.printArray(data, v.getName(), System.out, null);
    }
    v = ncf.findVariable("numChildren");
    if (v != null) {
      Array data = v.read();
      NCdump.printArray(data, v.getName(), System.out, null);
    }
    System.out.println();

    v = ncf.findVariable("record");
    //ucar.nc2.Variable v = ncf.findVariable("Latitude");
    //ucar.nc2.Variable v = ncf.findVariable("time");
    //System.out.println();
    //System.out.println( v.toString());

    if (v instanceof Structure) {
      Structure s = (Structure) v;
      StructureDataIterator iter = s.getStructureIterator();
      int count = 0;
      PrintWriter pw = new PrintWriter( System.out);
      while (iter.hasNext()) {
        System.out.println("record "+count);
        NCdumpW.printStructureData(pw, iter.next());
        count++;
      }
      Array data = v.read();
      NCdump.printArray(data, "record", System.out, null);
    } else {
      Array data = v.read();
      int[] length = data.getShape();
      System.out.println();
      System.out.println("v2 length =" + length[0]);

      IndexIterator ii = data.getIndexIterator();
      for (; ii.hasNext();) {
        System.out.println(ii.getFloatNext());
      }
    }
    ncf.close();  */
  }
}