/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import java.io.*;

/**
 * A vector of BaseType.
 *
 * @author jehamby
 * @version $Revision: 15901 $
 * @see PrimitiveVector
 */
public class BaseTypePrimitiveVector extends PrimitiveVector
{
    /**
     * the array of BaseType values.
     */
    private BaseType vals[];

    /**
     * Constructs a new BaseTypePrimitiveVector.
     *
     * @param var the template BaseType to use.
     */
    public BaseTypePrimitiveVector(BaseType var) {
        super(var);
    }

    /**
     * Returns a clone of this BaseTypePrimitiveVector.  A deep
     * copy is performed on all data inside the variable.
     *
     * @return a clone of this BaseTypePrimitiveVector.
     */
    public Object clone() {
        BaseTypePrimitiveVector v = (BaseTypePrimitiveVector) super.clone();
        if (vals != null) {
            v.vals = new BaseType[vals.length];
            System.arraycopy(vals, 0, v.vals, 0, vals.length);
        }
        return v;
    }

    /**
     * Returns the number of elements in the array.
     *
     * @return the number of elements in the array.
     */
    public int getLength() {
        return vals.length;
    }

    /**
     * Sets the number of elements in the array.  Allocates a new primitive
     * array of the desired size.  Note that if this is called multiple times,
     * the old array and its contents will be lost.
     *

     * Only called inside of deserialize method or in derived
     * classes on server.
     *
     * @param len the number of elements in the array.
     */
    public void setLength(int len) {
        vals = new BaseType[len];
    }

    /**
     * Return the i'th value as a BaseType.
     *
     * @param i the index of the value to return.
     * @return the i'th value.
     */
    public final BaseType getValue(int i) {
        return vals[i];
    }

    /**
     * Set the i'th value of the array.
     *
     * @param i      the index of the value to set.
     * @param newVal the new value.
     */
    public final void setValue(int i, BaseType newVal) {
        vals[i] = newVal;
        BaseType parent = (BaseType)getTemplate().getParent();
        vals[i].setParent(parent);
    }

    /**
     * Prints the value of all variables in this vector.  This
     * method is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *
     * @param os    the PrintWriter on which to print the value.
     * @param space this value is passed to the printDecl method,
     *              and controls the leading spaces of the output.
     * @see BaseType#printVal(PrintWriter, String, boolean)
     */
    public final void printVal(PrintWriter os, String space) {
        int len = vals.length;
        for (int i = 0; i < len - 1; i++) {
            vals[i].printVal(os, "", false);
            os.print(", ");
        }
        // print last value, if any, without trailing comma
        if (len > 0) {
            vals[len - 1].printVal(os, "", false);
        }
    }

    /**
     * Prints the value of a single variable in this vector.
     * method is used by DArray's printVal method.
     *
     * @param os    the PrintWriter on which to print the value.
     * @param index the index of the variable to print.
     * @see DArray#printVal(PrintWriter, String, boolean)
     */
    public void printSingleVal(PrintWriter os, int index) {
        vals[index].printVal(os, "", false);
    }

    /**
     * Reads data from a DataInputStream. This method is only used
     * on the client side of the OPeNDAP client/server connection.
     *
     * @param source   a DataInputStream to read from.
     * @param sv       The ServerVersion returned by the server.
     *                 (used by DSequence to determine which protocol version was
     *                 used).
     * @param statusUI The StatusUI object to use for GUI updates
     *                 and user cancellation notification (may be null).
     * @throws DataReadException when invalid data is read, or if the user
     *                           cancels the download.
     * @throws EOFException      if EOF is found before the variable is completely
     *                           deserialized.
     * @throws IOException       thrown on any other InputStream exception.
     * @see ClientIO#deserialize(DataInputStream, ServerVersion, StatusUI)
     */
    public synchronized void deserialize(DataInputStream source,
                                         ServerVersion sv,
                                         StatusUI statusUI)
            throws IOException, EOFException, DataReadException {
        for (int i = 0; i < vals.length; i++) {
            // create new variable from template
            vals[i] = (BaseType) getTemplate().clone();
            ((ClientIO) vals[i]).deserialize(source, sv, statusUI);
            if (statusUI != null && statusUI.userCancelled())
                throw new DataReadException("User cancelled");
        }
    }

    /**
     * Writes data to a DataOutputStream. This method is used
     * primarily by GUI clients which need to download OPeNDAP data, manipulate
     * it, and then re-save it as a binary file.
     *
     * @param sink a DataOutputStream to write to.
     * @throws IOException thrown on any OutputStream
     *                     exception.
     */
    public void externalize(DataOutputStream sink) throws IOException {
        for (int i = 0; i < vals.length; i++) {
            // LogStream.out.println("\t\t\tI AM THE WALRUS!");
            ((ClientIO) vals[i]).externalize(sink);
        }
    }

    /**
     * Write a subset of the data to a DataOutputStream.
     *
     * @param sink    a DataOutputStream to write to.
     * @param start  starting index (i=start)
     * @param stop   ending index (i<=stop)
     * @param stride index stride (i+=stride)
     * @throws IOException thrown on any OutputStream exception.
     */
    public void externalize(DataOutputStream sink, int start, int stop, int stride) throws IOException {
        for (int i = start; i <= stop; i += stride)
            ((ClientIO) vals[i]).externalize(sink);
    }

    /**
     * Returns (a reference to) the internal storage for this PrimitiveVector
     * object.
     *
WARNING:

     * Because this method breaks encapsulation rules the user must beware!
     * If we (the OPeNDAP prgramming team) choose to change the internal
     * representation(s) of these types your code will probably break.
     *

     * This method is provided as an optimization to eliminate massive
     * copying of data.
     *
     * @return The internal array of BaseTypes.
     */
    public Object getInternalStorage() {
        return (vals);
    }

    /**
     * Set the internal storage for PrimitiveVector.
     *
WARNING:

     * Because this method breaks encapsulation rules the user must beware!
     * If we (the OPeNDAP prgramming team) choose to change the internal
     * representation(s) of these types your code will probably break.
     *

     * This method is provided as an optimization to eliminate massive
     * copying of data.
     */
    public void setInternalStorage(Object o) {
        vals = (BaseType []) o;
    }
     /**
     * Returns the OPeNDAP type name of the class instance as a String.
     *
     * @return the OPeNDAP type name of the class instance as a String.
     */
    public String getTypeName() {
        return "BaseTypePrimitiveVector";
    }
/**
     * Prints the value of the variable, with its declaration.  This
     * function is primarily intended for debugging OPeNDAP applications and
     * text-based clients such as geturl.
     *
     * @param os           the PrintWriter on which to print the value.
     * @param space        this value is passed to the printDecl method,
     *                     and controls the leading spaces of the output.
     * @param print_decl_p a boolean value controlling whether the
     *                     variable declaration is printed as well as the value.
     */
     public void printVal(PrintWriter os, String space,
                                  boolean print_decl_p) {}

     /**
     * Create a new primitive vector using a subset of the data.
     *
     * @param start  starting index (i=start)
     * @param stop   ending index (i<=stop)
     * @param stride index stride (i+=stride)
     * @return new primitive vector, of type BaseTypePrimitiveVector.
     */
    public PrimitiveVector subset(int start, int stop, int stride) {
        BaseTypePrimitiveVector n = new BaseTypePrimitiveVector(getTemplate());
        stride = Math.max(stride, 1);
        stop = Math.max(start, stop);
        int length = 1 + (stop - start) / stride;
        n.setLength(length);

        int count = 0;
        for (int i = start; i <= stop; i += stride) {
            n.setValue(count, vals[i]);
            count++;
        }
        return n;
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.dataset;

import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.CF;

import java.util.*;

/**
 * Implementation of Enhancements for coordinate systems and standard attribute handling.
 * Factored out so that it can be used as a 'mixin' in VariablesDS and StructureDS.
 *
 * @author caron
 */
class EnhancementsImpl implements Enhancements {
  private Variable forVar;
  private String desc, units;
  private List<CoordinateSystem> coordSys; // dont allocate unless its used

  /**
   * Constructor when there's no underlying, existing Variable.
   * You can access units, description and coordSys.
   * All missing and scale/offset flags are false.
   * @param forVar the Variable to decorate.
   * @param units set unit string.
   * @param desc set description.
   */
  EnhancementsImpl( Variable forVar, String units, String desc) {
    this.forVar = forVar;
    this.units = units;
    this.desc = desc;
  }

  /**
   * Constructor.
   * @param forVar the Variable to decorate.
   */
  EnhancementsImpl( Variable forVar) {
    this.forVar = forVar;
  }

  /**
   * Get the list of Coordinate Systems for this variable.
   * Normally this is empty unless you use ucar.nc2.dataset.NetcdfDataset.
   * @return list of type ucar.nc2.dataset.CoordinateSystem; may be empty not null.
   */
  public List<CoordinateSystem> getCoordinateSystems() {
    return (coordSys == null) ? new ArrayList<>(0) : coordSys;
  }

  /** Add a CoordinateSystem to the dataset. */
  public void addCoordinateSystem( CoordinateSystem cs){
    if (cs == null)
      throw new RuntimeException("Attempted to add null CoordinateSystem to var " + forVar.getFullName());

    if (coordSys == null) coordSys = new ArrayList<>(5);
    coordSys.add(cs);
  }

  public void removeCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
    if (coordSys != null)
      coordSys.remove( p0);
  }

  /** Set the Description for this Variable.
   * @param desc description
   */
  public void setDescription(String desc) { this.desc = desc; }

  /** Get the description of the Variable.
   *  Default is to look for attributes in this order: CDM.LONG_NAME, "description", "title", "standard_name".
   */
  public String getDescription() {
    if ((desc == null) && (forVar != null)) {
      Attribute att = forVar.findAttributeIgnoreCase( CDM.LONG_NAME);
      if ((att != null) && att.isString())
        desc = att.getStringValue();

      if (desc == null) {
        att = forVar.findAttributeIgnoreCase( "description");
        if ((att != null) && att.isString())
          desc = att.getStringValue();
      }

      if (desc == null) {
        att = forVar.findAttributeIgnoreCase( CDM.TITLE);
        if ((att != null) && att.isString())
          desc = att.getStringValue();
      }

      if (desc == null) {
        att = forVar.findAttributeIgnoreCase(CF.STANDARD_NAME);
        if ((att != null) && att.isString())
          desc = att.getStringValue();
      }
    }
    return (desc == null) ? null : desc.trim();
  }

  /** Set the Unit String for this Variable. Default is to use the CDM.UNITS attribute.
   * @param units  unit string
   */
  public void setUnitsString( String units) {
    this.units = units;
    forVar.addAttribute( new Attribute(CDM.UNITS, units));
  }

  /**
   * Get the Unit String for the Variable. May be set explicitly, else look for attribute CDM.UNITS.
   * @return the Unit String for the Variable, or null if none.
   */
  public String getUnitsString() {
    String result = units;
    if ((result == null) && (forVar != null)) {
      Attribute att = forVar.findAttribute( CDM.UNITS);
      if (att == null) att = forVar.findAttributeIgnoreCase( CDM.UNITS);
      if ((att != null) && att.isString())
        result = att.getStringValue();
    }
    return (result == null) ? null : result.trim();
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.core;

import com.coverity.security.Escape;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import thredds.client.catalog.Access;
import thredds.client.catalog.CatalogRef;
import thredds.client.catalog.Dataset;
import thredds.featurecollection.FeatureCollectionCache;
import thredds.server.admin.DebugCommands;
import thredds.server.catalog.*;
import thredds.server.catalog.builder.ConfigCatalogBuilder;
import thredds.server.catalog.tracker.*;
import thredds.server.config.TdsContext;
import thredds.server.config.ThreddsConfig;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.util.Counters;
import ucar.util.prefs.PreferencesExt;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.*;
import java.util.*;

/**
 * Reads in the Config catalogs on startup, and if triggered.
 * Spring managed auto-wired.
 *
 * @author caron
 * @since 3/21/2015
 */
@Component
public class ConfigCatalogInitialization {
  static private final org.slf4j.Logger logCatalogInit = org.slf4j.LoggerFactory.getLogger(ConfigCatalogInitialization.class);
  static private final String ERROR = "*** ERROR: ";
  static private final boolean show = true;
  static private final ReadMode defaultReadMode = ReadMode.check;

  @Autowired
  private TdsContext tdsContext;  // used for  getContentDirectory, contextPath

  @Autowired
  private AllowedServices allowedServices;

  @Autowired
  private ConfigCatalogCache ccc;

  @Autowired
  private DataRootManager dataRootManager;

  @Autowired
  private DatasetManager datasetManager;

  @Autowired
  private DebugCommands debugCommands;

  @Autowired
  private FeatureCollectionCache fcCache;

  ///////////////////////////////////////////////////////
  public enum ReadMode {always, check, triggerOnly;
    static public ReadMode get(String name) {
      for (ReadMode mode : values()) {
        if (mode.name().equalsIgnoreCase(name)) return mode;
      }
      return null;
    }
  }

  private PreferencesExt prefs;
  private long readNow;
  private long trackerNumber = 1;
  private long nextCatId = 1;
  private int numberCatalogs = 10;
  private File contentRootPath;    // ${tds.content.root.path}.
  private String contextPath;      // thredds
  private String trackerDir;       // the tracker "databases" are kept in this directory
  private long maxDatasets;        // chronicle limit

   // on reread, construct new objects, so cant be spring beans
  private DataRootPathMatcher dataRootPathMatcher;
  private DataRootTracker dataRootTracker;
  private DatasetTracker datasetTracker;

  // temporary, discard after init
  private CatalogTracker catalogTracker;
  private Set<String> catPathMap;          // Hash of paths, to look for duplicate catalogs
  private Map<String, String> fcNameMap;   // Hash of featureCollection ids, to look for duplicates
  private List<String> rootCatalogKeys;    // needed ??

  // track stats
  private DatasetTracker.Callback callback;

  public ConfigCatalogInitialization() {
  }

  public synchronized void setTrackerDir(String trackerDir) {
    this.trackerDir = trackerDir;
  }

  public void setMaxDatasetToTrack(long maxDatasets) {
    this.maxDatasets = maxDatasets;
  }

  // called from TdsInit on spring-managed auto-wired bean
  public synchronized void init(ReadMode readMode, PreferencesExt prefs) {
    if (readMode == null)
      readMode = defaultReadMode;
    this.prefs = prefs;
    trackerNumber = prefs.getLong("trackerNumber", 1);
    numberCatalogs = prefs.getInt("numberCatalogs", 10);
    nextCatId = prefs.getLong("nextCatId", 1);

    makeDebugActions();
    this.contentRootPath = this.tdsContext.getThreddsDirectory();
    this.contextPath = tdsContext.getContextPath();

    reread(readMode, true);
  }

  // called from init() and from trigger controller
  public synchronized boolean reread(ReadMode readMode, boolean isStartup) {
    readNow = System.currentTimeMillis();
    logCatalogInit.info("=========================================================================================\n"+
                    "ConfigCatalogInitialization readMode={} isStartup={}", readMode, isStartup);
    catPathMap = new HashSet<>();
    fcNameMap = new HashMap<>();
    if (ccc != null) ccc.invalidateAll(); // remove anything in cache
    if (fcCache != null) fcCache.invalidateAll(); // remove anything in cache

    if (!isStartup && readMode == ReadMode.always) trackerNumber++;  // must write a new database if TDS is already running and rereading all
    if (!isDebugMode || this.datasetTracker == null)
      this.datasetTracker = new DatasetTrackerChronicle(trackerDir, maxDatasets, trackerNumber);

    boolean databaseAlreadyExists = datasetTracker.exists(); // detect if tracker database exists
    if (!databaseAlreadyExists) {
      readMode = ReadMode.always;
      logCatalogInit.info("ConfigCatalogInitializion datasetTracker database does not exist, set readMode to=" + readMode);
    }

    if (this.callback == null)
      this.callback = new StatCallback(readMode);

    // going to reread global services
    allowedServices.clearGlobalServices();

    switch (readMode) {
      case always:
        // if the database already exists, we need to close it
        // before we reinit
        if (databaseAlreadyExists) {
          logCatalogInit.info("ConfigCatalogInitializion datasetTracker database already exists - closing it before reinitialization.");
          try {
            this.datasetTracker.close();
          } catch (IOException e) {
            logCatalogInit.error("There was an error closing the datasetTracker database.", e);
          }
          this.datasetTracker.reinit();
        }
        this.catalogTracker = new CatalogTracker(trackerDir, true, numberCatalogs, nextCatId);
        this.dataRootTracker = new DataRootTracker(trackerDir, true, callback);
        this.dataRootPathMatcher = new DataRootPathMatcher(ccc, dataRootTracker);  // starting over
        readRootCatalogs(readMode);
        break;

      case check:
        this.catalogTracker = new CatalogTracker(trackerDir, false, numberCatalogs, nextCatId);        // use existing catalog list
        this.dataRootTracker = new DataRootTracker(trackerDir, false, callback);      // use existing data roots
        this.dataRootPathMatcher = new DataRootPathMatcher(ccc, dataRootTracker);
        readRootCatalogs(readMode);           // read just roots to get global services
        checkExistingCatalogs(readMode);
        break;

      case triggerOnly:
        this.catalogTracker = new CatalogTracker(trackerDir, false, numberCatalogs, nextCatId);               // use existing catalog list
        this.dataRootTracker = new DataRootTracker(trackerDir, false, callback);             // use existing data roots
        this.dataRootPathMatcher = new DataRootPathMatcher(ccc, dataRootTracker);
        readRootCatalogs(readMode);           // read just roots to get global services
        break;
    }

    numberCatalogs = catalogTracker.size();
    nextCatId = catalogTracker.getNextCatId();
    if (prefs != null) {
      prefs.putLong("trackerNumber", trackerNumber);
      prefs.putLong("nextCatId", nextCatId);
      prefs.putInt("numberCatalogs", numberCatalogs);
    }
    callback.finish();
    logCatalogInit.info("\nConfigCatalogInitializion stats\n" + callback);

    try {
      datasetTracker.save();
      catalogTracker.save();
      dataRootTracker.save();
    } catch (IOException e) {
      // e.printStackTrace();
      logCatalogInit.error("datasetTracker.save() failed", e);
    }

    // heres where we may be doing a switcheroo in a running TDS
    if (dataRootManager != null)
      dataRootManager.setDataRootPathMatcher(dataRootPathMatcher);
    if (datasetManager != null)
      datasetManager.setDatasetTracker(datasetTracker);

    // cleanup old version of the database
    if (!isStartup && readMode == ReadMode.always) {
      DatasetTrackerChronicle.cleanupBefore(trackerDir, trackerNumber);
    }

    long took = System.currentTimeMillis() - readNow;
    logCatalogInit.info("ConfigCatalogInitializion finished took={} msecs", took);

    // cleanup
    catPathMap = null;
    fcNameMap = null;
    catalogTracker = null;

    return true; // ok
  }

  private void readRootCatalogs(ReadMode readMode) {
    rootCatalogKeys = new ArrayList<>();
    rootCatalogKeys.add("catalog.xml"); // always first
    // add any others listed in ThreddsConfig
    for (String location : ThreddsConfig.getRootList("catalogRoot"))
      rootCatalogKeys.add( location );
    logCatalogInit.info("ConfigCatalogInit: initializing " + rootCatalogKeys.size() + " root catalogs.");

    // all root catalogs are checked
    for (String pathname : rootCatalogKeys) {
      try {
        pathname = StringUtils.cleanPath(pathname);
        logCatalogInit.info( "Checking catalogRoot = " + pathname);
        checkCatalogToRead(readMode, pathname, true, 0);
      } catch (Throwable e) {
        logCatalogInit.error(ERROR + "initializing catalog " + pathname + "; " + e.getMessage(), e);
      }
    }
  }

  private void checkExistingCatalogs(ReadMode readMode) {
    for (CatalogExt catalogExt : catalogTracker.getCatalogs()) {
      if (catalogExt.isRoot()) continue; // already read in

      String pathname = catalogExt.getCatRelLocation();
      try {
        logCatalogInit.info("\n**************************************\nCatalog init " + pathname + "[" + CalendarDate.present() + "]");
        pathname = StringUtils.cleanPath(pathname);
        checkCatalogToRead(readMode, pathname, catalogExt.isRoot(), catalogExt.getLastRead());
      } catch (Throwable e) {
        logCatalogInit.error(ERROR + "initializing catalog " + pathname + "; " + e.getMessage(), e);
      }
    }
  }

  // decide if we need to read this catalog or not. if yes, follow any catrefs
  // catalogRelpath must be relative to rootDir
  private void checkCatalogToRead(ReadMode readMode, String catalogRelPath, boolean isRoot, long lastRead) throws IOException {
    if (exceedLimit) return;

    catalogRelPath = StringUtils.cleanPath(catalogRelPath);
    File catalogFile = new File(this.contentRootPath, catalogRelPath);
    if (!catalogFile.exists()) {
      catalogTracker.removeCatalog(catalogRelPath);
      logCatalogInit.error(ERROR + "initCatalog(): Catalog [" + catalogRelPath + "] does not exist.");
      return;
    }
    long lastModified = catalogFile.lastModified();
    if (!isRoot && readMode != ReadMode.always && lastModified < lastRead) return; // skip catalogs that havent changed
    if (!isRoot && readMode == ReadMode.triggerOnly) return;                    // skip non-root catalogs for trigger only
    if (show) System.out.printf("initCatalog %s%n", catalogRelPath);

    // make sure we havent already read it
    if (catPathMap.contains(catalogRelPath)) {
      logCatalogInit.error(ERROR + "initCatalog(): Catalog [" + catalogRelPath + "] already seen, possible loop (skip).");
      return;
    }
    catPathMap.add(catalogRelPath);
    Set<String> idSet = new HashSet<>();  // look for unique ids

    // if (logCatalogInit.isDebugEnabled()) logCatalogInit.debug("initCatalog {} -> {}", path, f.getAbsolutePath());

    // read it
    ConfigCatalog cat = readCatalog(catalogRelPath, catalogFile.getPath());
    if (cat == null) {
      logCatalogInit.error(ERROR + "initCatalog(): failed to read catalog <" + catalogFile.getPath() + ">.");
      return;
    }
    long catId = catalogTracker.put(new CatalogExt(0, catalogRelPath, isRoot, readNow));

    if (isRoot) {
      if (ccc != null) ccc.put(catalogRelPath, cat);
      allowedServices.addGlobalServices(cat.getServices());
      if (readMode == ReadMode.triggerOnly) return;                    // thats all we need
    }

    if (callback != null) callback.hasCatalogRef(cat);

    // look for datasetRoots
    for (DatasetRootConfig p : cat.getDatasetRoots())
      dataRootPathMatcher.addRoot(p, catalogRelPath, readMode == ReadMode.always); // check for duplicates on complete reread

    if (callback == null) {   // LOOK WTF?
      List<String> disallowedServices = allowedServices.getDisallowedServices(cat.getServices());
      if (!disallowedServices.isEmpty()) {
        allowedServices.getDisallowedServices(cat.getServices());
        logCatalogInit.error(ERROR + "initCatalog(): declared services: " + Arrays.toString(disallowedServices.toArray()) + " in catalog: " + catalogFile.getPath() + " are disallowed in threddsConfig file");
      }
    }

    // look for dataRoots in datasetScans and featureCollections
    dataRootPathMatcher.extractDataRoots(catalogRelPath, cat.getDatasetsLocal(), readMode == ReadMode.always, fcNameMap);

    // get the directory path, reletive to the rootDir
    int pos = catalogRelPath.lastIndexOf("/");
    String dirPath = (pos > 0) ? catalogRelPath.substring(0, pos + 1) : "";
    processDatasets(catId, readMode, dirPath, cat.getDatasetsLocal(), idSet);     // recurse

    // look for catalogScans
    for (CatalogScan catScan : cat.getCatalogScans()) {
      if (exceedLimit) return;
      Path relLocation = Paths.get(dirPath, catScan.getLocation());
      Path absLocation = Paths.get(catalogFile.getParent(), catScan.getLocation());
      // if (catalogWatcher != null) catalogWatcher.registerAll(absLocation);
      readCatsInDirectory(readMode, relLocation.toString(), absLocation);
    }
  }

  /**
   * Does the actual work of reading a catalog.
   *
   * @param catalogRelPath            reletive path starting from content root
   * @param catalogFullPath           absolute location on disk
   * @return the Catalog, or null if failure
   */
  private ConfigCatalog readCatalog(String catalogRelPath, String catalogFullPath)  {
    URI uri;
    try {
      // uri = new URI("file:" + StringUtil2.escape(catalogFullPath, "/:-_.")); // needed ?
      uri = new URI(this.contextPath + "/catalog/" + catalogRelPath);
    } catch (URISyntaxException e) {
      logCatalogInit.error(ERROR + "readCatalog(): URISyntaxException=" + e.getMessage());
      return null;
    }

    ConfigCatalogBuilder builder = new ConfigCatalogBuilder();
    try {
      // read the catalog
      logCatalogInit.info("-------readCatalog(): path=" + catalogRelPath);
      ConfigCatalog cat = (ConfigCatalog) builder.buildFromLocation(catalogFullPath, uri);
      if (builder.hasFatalError()) {
        logCatalogInit.error(ERROR + "   invalid catalog -- " + builder.getErrorMessage());
        return null;
      }

      if (builder.getErrorMessage().length() > 0)
        logCatalogInit.debug(builder.getErrorMessage());

      return cat;

    } catch (Throwable t) {
      logCatalogInit.error(ERROR + "  Exception on catalog=" + catalogFullPath + " " + t.getMessage() + "\n log=" + builder.getErrorMessage(), t);
      return null;
    }
  }

  // dirPath = the directory path, reletive to the rootDir
  private void processDatasets(long catId, ReadMode readMode, String dirPath, List<Dataset> datasets, Set<String> idMap) throws IOException {
    if (exceedLimit) return;

    for (Dataset ds : datasets) {
      if (datasetTracker.trackDataset(catId, ds, callback)) countDatasets++;
      if (maxDatasetsProcess > 0 && countDatasets > maxDatasetsProcess) exceedLimit = true;

      // look for duplicate ids
      String id = ds.getID();
      if (id != null) {
        if (idMap.contains(id)) {
          logCatalogInit.error(ERROR + "Duplicate id on  '" + ds.getName() + "' id= '" + id + "'");
        } else {
          idMap.add(id);
        }
      }

      if ((ds instanceof DatasetScan) || (ds instanceof FeatureCollectionRef)) continue;
      if (ds instanceof CatalogScan) continue;

      if (ds instanceof CatalogRef) { // follow catalog refs
        CatalogRef catref = (CatalogRef) ds;
        String href = catref.getXlinkHref();
        // if (logCatalogInit.isDebugEnabled()) logCatalogInit.debug("  catref.getXlinkHref=" + href);

        // Check that catRef is relative
        if (!href.startsWith("http:")) {
          // Clean up relative URLs that start with "./"
          if (href.startsWith("./")) {
            href = href.substring(2);
          }

          String path;
          String contextPathPlus = this.contextPath + "/";
          if (href.startsWith(contextPathPlus)) {
            path = href.substring(contextPathPlus.length()); // absolute starting from content root
          } else if (href.startsWith("/")) {
            // Drop the catRef because it points to a non-TDS served catalog.
            logCatalogInit.error(ERROR + "Skipping catalogRef <xlink:href=" + href + ">. Reference is relative to the server outside the context path [" + contextPathPlus + "]. " +
                    "Parent catalog info: Name=\"" + catref.getParentCatalog().getName() + "\"; Base URI=\"" + catref.getParentCatalog().getUriString() + "\"; dirPath=\"" + dirPath + "\".");
            continue;
          } else {
            path = dirPath + href;  // reletive starting from current directory
          }

          CatalogExt ext = catalogTracker.get(path);
          long lastRead = (ext == null) ? 0 : ext.getLastRead();
          checkCatalogToRead(readMode, path, false, lastRead);
        }

      } else {
        // recurse through nested datasets
        processDatasets(catId, readMode, dirPath, ds.getDatasetsLocal(), idMap);
      }
    }
  }

  // dirPath is the directory relative to rootDir, directory is absolute
  private void readCatsInDirectory(ReadMode readMode, String dirPath, Path directory) throws IOException {
    if (exceedLimit) return;

     // do any catalogs first
    try (DirectoryStream<Path> ds = Files.newDirectoryStream(directory, "*.xml")) {
      for (Path p : ds) {
        if (!Files.isDirectory(p)) {
          // path must be relative to rootDir
          String filename = p.getFileName().toString();
          String path = dirPath.length() == 0 ? filename :  dirPath + "/" + filename;  // reletive starting from current directory

          CatalogExt ext = catalogTracker.get(path);
          long lastRead = (ext == null) ? 0 : ext.getLastRead();
          checkCatalogToRead(readMode, path, false, lastRead);
        }
      }
    }

    // now recurse into the directory
    try (DirectoryStream<Path> ds = Files.newDirectoryStream(directory)) {
       for (Path dir : ds) {
         if (Files.isDirectory(dir)) {
           String dirPathChild = dirPath + "/" + dir.getFileName().toString();  // reletive starting from current directory
           readCatsInDirectory(readMode, dirPathChild, dir);
         }
       }
     }
   }

  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////

  public void makeDebugActions() {
    DebugCommands.Category debugHandler = debugCommands.findCategory("Catalogs");
    DebugCommands.Action act;

     act = new DebugCommands.Action("showCatalogExt", "Show known catalogs") {
       public void doAction(DebugCommands.Event e) {
         e.pw.printf("numberCatalogs=%d nextCatId=%d%n", numberCatalogs, nextCatId);
         e.pw.printf("%nid  root  lastRead     path%n");

         CatalogTracker catalogTracker = new CatalogTracker(trackerDir, false, numberCatalogs, 0);
         for (CatalogExt cat : catalogTracker.getCatalogs()) {
           e.pw.printf("%3d: %5s %s %s%n", cat.getCatId(), cat.isRoot(), CalendarDate.of(cat.getLastRead()), cat.getCatRelLocation());
         }
       }
     };
     debugHandler.addAction(act);

    act = new DebugCommands.Action("showRoots", "Show root catalogs") {
      public void doAction(DebugCommands.Event e) {
        StringBuilder sbuff = new StringBuilder();
        synchronized (ConfigCatalogInitialization.this) {
          for (String catPath : rootCatalogKeys) {
            sbuff.append(" catalog= ").append(catPath).append("\n");
            //String filename = StringUtil2.unescape(cat.getUriString());
            //sbuff.append(" from= ").append(filename).append("\n");
          }
        }
        e.pw.println();
        e.pw.println(Escape.html(sbuff.toString()));
      }
    };
    debugHandler.addAction(act);

    act = new DebugCommands.Action("showStats", "Show catalog initialization stats") {
      public void doAction(DebugCommands.Event e) {
        if (callback != null)
          e.pw.printf("%n%s%n", Escape.html(callback.toString()));
        else
          e.pw.printf("N/A%n");
      }
    };
    debugHandler.addAction(act);
  }

  /////////////////////////////////////////////////////////
  static class Stats {
    int catrefs;
    int datasets, trackedDatasets;
    int dataRoot, dataRootFc, datasetScan, datasetRoot, catalogScan;
    int ncml, ncmlOne;
    int restrict;
    Counters counters = new Counters();

    public Stats() {
      counters.add("restrict");
      counters.add("nAccess");
      counters.add("serviceType");
      counters.add("ncmlAggSize");
    }

    String show(Formatter f) {
      f.format("         catalogs=%d%n", catrefs);
      f.format("         datasets=%d%n", datasets);
      f.format("  trackedDatasets=%d%n", trackedDatasets);
      f.format("           restrict=%d%n", restrict);
      f.format("            hasNcml=%d%n%n", ncml);

      f.format("      dataRoot=%d%n", dataRoot);
      f.format("    featCollect=%d%n", dataRootFc);
      f.format("    datasetScan=%d%n", datasetScan);
      f.format("    catalogScan=%d%n", catalogScan);
      f.format("    datasetRoot=%d%n%n", datasetRoot);

      f.format("DatasetExt.total_count %d%n", DatasetExt.total_count);
      f.format("DatasetExt.total_nbytes %d%n", DatasetExt.total_nbytes);
      float avg = DatasetExt.total_count == 0 ? 0 : ((float) DatasetExt.total_nbytes) / DatasetExt.total_count;
      f.format("DatasetExt.avg_nbytes %5.0f%n", avg);

      counters.show(f);
      return f.toString();
    }
  }

  static public class StatCallback implements DatasetTracker.Callback {
    ReadMode readMode;
    Stats stat2;
    long start = System.currentTimeMillis();
    double took;

    public StatCallback(ReadMode readMode) {
      this.readMode = readMode;
      stat2 = new Stats();
    }

    @Override
    public void finish() {
      took = (System.currentTimeMillis() - start) / 1000.0;
    }

    @Override
    public void hasDataRoot(DataRootExt dataRoot) {
      stat2.dataRoot++;
      switch (dataRoot.getType()) {
        case featureCollection:
          stat2.dataRootFc++;
          break;
        case catalogScan:
          stat2.catalogScan++;
          break;
        case datasetScan:
          stat2.datasetScan++;
          break;
        case datasetRoot:
          stat2.datasetRoot++;
          break;
      }
    }

    @Override
    public void hasDataset(Dataset ds) {
      stat2.datasets++;
      List<Access> access = ds.getAccess();
      stat2.counters.count("nAccess", access.size());
      for (Access acc : access)
        if (acc.getService() != null) stat2.counters.count("serviceType", acc.getService().toString());
    }

    @Override
    public void hasTrackedDataset(Dataset ds) {
      stat2.trackedDatasets++;
    }

    @Override
    public void hasNcml(Dataset ds) {
      stat2.ncml++;
      org.jdom2.Element netcdfElem = ds.getNcmlElement();
      org.jdom2.Element agg =  netcdfElem.getChild("aggregation", thredds.client.catalog.Catalog.ncmlNS);
      if (agg == null) return;
      List<org.jdom2.Element> nested =  agg.getChildren("netcdf", thredds.client.catalog.Catalog.ncmlNS);
      if (nested == null) return;
      if (nested.size() == 1)
        stat2.ncmlOne++;
      // look for nested ncml - count how many
      stat2.counters.count("ncmlAggSize", nested.size());
    }

    @Override
    public void hasRestriction(Dataset ds) {
      stat2.restrict++;
      String restrict = ds.getRestrictAccess();
      if (restrict != null) stat2.counters.count("restrict", restrict);
    }

    @Override
    public void hasCatalogRef(ConfigCatalog dd) {
      stat2.catrefs++;
    }

    @Override
    public String toString() {
      Formatter f = new Formatter();
      f.format("ConfigCatalogInitialization started %s took %f secs using readMode=%s%n", CalendarDate.of(start), took, readMode);
      return stat2.show(f);
    }
  }

  /////////////////////////////////////////////////////////////////////////////////
  // debugging mode, to test outside of tomcat/spring

  private boolean isDebugMode;
  private long countDatasets = 0;
  private long maxDatasetsProcess;
  private boolean exceedLimit;

  // used from outside of tomcat/spring for testing
  public ConfigCatalogInitialization(ReadMode readMode, File contentRootPath, String trackerDir, DatasetTracker datasetTracker,
                                     AllowedServices allowedServices, DatasetTracker.Callback callback, long maxDatasetsProcess) throws IOException {
    this.contentRootPath = contentRootPath;
    this.contextPath = "/thredds";
    this.trackerDir = trackerDir != null ? trackerDir : new File(contentRootPath, "cache/catalog").getPath();
    this.datasetTracker = datasetTracker;
    this.allowedServices = allowedServices;
    this.callback = callback;
    this.maxDatasetsProcess = maxDatasetsProcess;
    this.isDebugMode = true;

    File trackerFile = new File(this.trackerDir);
    if (!trackerFile.exists()) {
      boolean ok = trackerFile.mkdirs();
      System.out.printf("ConfigCatalogInitialization make tracker directory '%s' make ok = %s%n", this.trackerDir, ok);
    }

    reread(readMode, true);
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.unidata.geoloc;

import ucar.nc2.util.Misc;

import java.util.Formatter;

/**
 * Standard implementation of LatLonPoint.
 * Longitude is always between -180 and +180 deg.
 * Latitude is always between -90 and +90 deg.
 *
 * @author Russ Rew
 * @author John Caron
 * @see LatLonPoint
 */
public class LatLonPointImpl implements LatLonPoint, java.io.Serializable {

  /**
   * Test if point lies between two longitudes, deal with wrapping.
   *
   * @param lon    point to test
   * @param lonBeg beginning longitude
   * @param lonEnd ending longitude
   * @return true if lon is between lonBeg and lonEnd.
   * @deprecated
   */
  static public boolean betweenLon(double lon, double lonBeg, double lonEnd) {
    lonBeg = lonNormal(lonBeg, lon);
    lonEnd = lonNormal(lonEnd, lon);
    return (lon >= lonBeg) && (lon <= lonEnd);
  }

  static public double getClockwiseDistanceTo(double from, double to) {
    double distance = to - from;
    while (distance < 0.0)
      distance += 360.0;
    return distance;
  }

  /**
   * put longitude into the range [-180, 180] deg
   *
   * @param lon lon to normalize
   * @return longitude in range [-180, 180] deg
   */
  static public double range180(double lon) {
    return lonNormal(lon);
  }

  /**
   * put longitude into the range [0, 360] deg
   *
   * @param lon lon to normalize
   * @return longitude into the range [0, 360] deg
   */
  static public double lonNormal360(double lon) {
    return lonNormal(lon, 180.0);
  }

  /**
   * put longitude into the range [center +/- 180] deg
   *
   * @param lon    lon to normalize
   * @param center center point
   * @return longitude into the range [center +/- 180] deg
   */
  static public double lonNormal(double lon, double center) {
    return center + Math.IEEEremainder(lon - center, 360.0);
  }

  /**
   * put longitude into the range [start, start+360] deg
   *
   * @param lon    lon to normalize
   * @param start starting point
   * @return longitude into the [start, start+360] deg
   */
  static public double lonNormalFrom(double lon, double start) {
    while (lon < start) lon += 360;
    while (lon > start+360) lon -= 360;
    return lon;
  }

  /**
   * Normalize the longitude to lie between +/-180
   *
   * @param lon east latitude in degrees
   * @return normalized lon
   */
  static public double lonNormal(double lon) {
    if ((lon < -180.) || (lon > 180.)) {
      return Math.IEEEremainder(lon, 360.0);
    } else {
      return lon;
    }
  }

  /**
   * Find difference (lon1 - lon2) normalized so that maximum value is += 180.
   * @param lon1 start
   * @param lon2 end
   * @return
   */
  static public double lonDiff(double lon1, double lon2) {
    return Math.IEEEremainder(lon1-lon2, 360.0);
  }

  /**
   * Normalize the latitude to lie between +/-90
   *
   * @param lat north latitude in degrees
   * @return normalized lat
   */
  static public double latNormal(double lat) {
    if (lat < -90.) {
      return -90.;
    } else if (lat > 90.) {
      return 90.;
    } else {
      return lat;
    }
  }

  /**
   * Make a nicely formatted representation of a latitude, eg 40.34N or 12.9S.
   *
   * @param lat       the latitude.
   * @param ndec      number of digits to right of decimal point
   * @return String representation.
   */
  static public String latToString(double lat, int ndec) {
    boolean is_north = (lat >= 0.0);
    if (!is_north)
      lat = -lat;

    String f = "%."+ndec+"f";

    Formatter latBuff = new Formatter();
    latBuff.format(f, lat);
    latBuff.format("%s", is_north ? "N" : "S");

    return latBuff.toString();
  }

  /**
   * Make a nicely formatted representation of a longitude, eg 120.3W or 99.99E.
   *
   * @param lon       the longitude.
   * @param ndec      number of digits to right of decimal point
   * @return String representation.
   */
  static public String lonToString(double lon, int ndec) {
    double wlon = lonNormal(lon);
    boolean is_east = (wlon >= 0.0);
    if (!is_east)
      wlon = -wlon;

    String f = "%."+ndec+"f";

    Formatter latBuff = new Formatter();
    latBuff.format(f, wlon);
    latBuff.format("%s", is_east ? "E" : "W");

    return latBuff.toString();
  }

  ///////////////////////////////////////////////////////////////////////////////////

  /**
   * East latitude in degrees, always +/- 90
   */
  protected double lat;

  /**
   * North longitude in degrees, always +/- 180
   */
  protected double lon;

  /**
   * Default constructor with values 0,0.
   */
  public LatLonPointImpl() {
    // Don't initialize by calling setLatitude() and setLongitude(), as those methods throw
    // UnsupportedOperationException in LatLonPointImmutable, and this constructor is (implicitly) invoked in the
    // LatLonPointImmutable constructors.
    this.lat = 0;
    this.lon = 0;
  }

  /**
   * Copy Constructor.
   *
   * @param pt point to copy
   */
  public LatLonPointImpl(LatLonPoint pt) {
    this(pt.getLatitude(), pt.getLongitude());
  }

  /**
   * Creates a LatLonPoint from component latitude and longitude values.
   * The longitude is adjusted to be in the range [-180.,180.].
   *
   * @param lat north latitude in degrees
   * @param lon east longitude in degrees
   */
  public LatLonPointImpl(double lat, double lon) {
    setLatitude(lat);
    setLongitude(lon);
  }

  /**
   * Returns the longitude, in degrees.
   *
   * @return the longitude, in degrees
   */
  public double getLongitude() {
    return lon;
  }

  /**
   * Returns the latitude, in degrees.
   *
   * @return the latitude, in degrees
   */
  public double getLatitude() {
    return lat;
  }

  /**
   * set lat, lon using values of pt
   *
   * @param pt point to use
   */
  public void set(LatLonPoint pt) {
    setLongitude(pt.getLongitude());
    setLatitude(pt.getLatitude());
  }

  /**
   * set lat, lon using double values
   *
   * @param lat lat value
   * @param lon lon value
   */
  public void set(double lat, double lon) {
    setLongitude(lon);
    setLatitude(lat);
  }

  /**
   * set lat, lon using float values
   *
   * @param lat lat value
   * @param lon lon value
   */
  public void set(float lat, float lon) {
    setLongitude((double) lon);
    setLatitude((double) lat);
  }

  /**
   * Set the longitude, in degrees. It is normalized to +/-180.
   *
   * @param lon east longitude in degrees
   */
  public void setLongitude(double lon) {
    this.lon = lonNormal(lon);
  }

  /**
   * Set the latitude, in degrees. Must lie beween +/-90
   *
   * @param lat north latitude in degrees
   */
  public void setLatitude(double lat) {
    this.lat = latNormal(lat);
  }

  @Override
  public boolean nearlyEquals(LatLonPoint that, double maxRelDiff) {
    boolean lonOk = Misc.nearlyEquals(that.getLongitude(), this.lon, maxRelDiff);
    if (!lonOk) {
      // We may be in a situation where "this.lon ≈ -180" and "that.lon ≈ +180", or vice versa.
      // Those longitudes are equivalent, but not "nearlyEquals()". So, we normalize them both to lie in
      // [0, 360] and compare them again.
      lonOk = Misc.nearlyEquals(lonNormal360(that.getLongitude()), lonNormal360(this.lon), maxRelDiff);
    }
    return lonOk && Misc.nearlyEquals(that.getLatitude(), this.lat, maxRelDiff);
  }

  // Exact comparison is needed in order to be consistent with hashCode().
  @Override
  public boolean equals(Object other) {
    if (this == other) {
      return true;
    }
    if (!(other instanceof LatLonPoint)) {
      return false;
    }
    LatLonPoint that = (LatLonPoint) other;
    if (Double.compare(that.getLatitude(), this.getLatitude()) != 0) {
      return false;
    }
    return Double.compare(that.getLongitude(), this.getLongitude()) == 0;
  }

  @Override
  public int hashCode() {
    int result;
    long temp;
    temp = Double.doubleToLongBits(lat);
    result = (int) (temp ^ (temp >>> 32));
    temp = Double.doubleToLongBits(lon);
    result = 31 * result + (int) (temp ^ (temp >>> 32));
    return result;
  }

  /**
   * Default string representation
   *
   * @return string representing this point
   */
  public String toString() {
    return toString(4);
  }

  /**
   * String representation in the form, eg 40.23N 105.1W
   *
   * @param sigDigits significant digits
   * @return String representation
   */
  public String toString(int sigDigits) {
    Formatter sbuff = new Formatter();
    sbuff.format("%s %s", latToString(lat, sigDigits), lonToString(lon, sigDigits));
    return sbuff.toString();
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.units;

import java.io.Serializable;

/**
 * Provides support for abstract base quantities (ex: length, time).
 *
 * @author Steven R. Emmerson
 */
public abstract class BaseQuantity implements Base, Comparable<BaseQuantity>,
		Serializable {
	private static final long						serialVersionUID	= 1L;

	/**
	 * The base quantity of amount of substance.
	 */
	public static final RegularBaseQuantity			AMOUNT_OF_SUBSTANCE;

	/**
	 * The base quantity of electric current.
	 */
	public static final RegularBaseQuantity			ELECTRIC_CURRENT;

	/**
	 * The base quantity of length.
	 */
	public static final RegularBaseQuantity			LENGTH;

	/**
	 * The base quantity of luminous intensity.
	 */
	public static final RegularBaseQuantity			LUMINOUS_INTENSITY;

	/**
	 * The base quantity of mass.
	 */
	public static final RegularBaseQuantity			MASS;

	/**
	 * The base quantity of plane angle.
	 */
	public static final SupplementaryBaseQuantity	PLANE_ANGLE;

	/**
	 * The base quantity of solid angle.
	 */
	public static final SupplementaryBaseQuantity	SOLID_ANGLE;

	/**
	 * The base quantity of themodynamic temperature.
	 */
	public static final RegularBaseQuantity			THERMODYNAMIC_TEMPERATURE;

	/**
	 * The base quantity of time.
	 */
	public static final RegularBaseQuantity			TIME;

	/**
	 * The unknown base quantity.
	 */
	public static final UnknownBaseQuantity			UNKNOWN;

	/**
	 * @serial
	 */
	private final String							name;						// never
	// null
	// or
	// empty

	/**
	 * @serial
	 */
	private final String							symbol;					// may
	// be
	// null
	// ;
	// never
	// empty

	static {
		AMOUNT_OF_SUBSTANCE = new RegularBaseQuantity("Amount of Substance",
				"N", true);
		ELECTRIC_CURRENT = new RegularBaseQuantity("Electric Current", "I",
				true);
		LENGTH = new RegularBaseQuantity("Length", "L", true);
		LUMINOUS_INTENSITY = new RegularBaseQuantity("Luminous Intensity", "J",
				true);
		MASS = new RegularBaseQuantity("Mass", "M", true);
		PLANE_ANGLE = new SupplementaryBaseQuantity("Plane Angle", null, true);
		SOLID_ANGLE = new SupplementaryBaseQuantity("Solid Angle", null, true);
		THERMODYNAMIC_TEMPERATURE = new RegularBaseQuantity(
				"Thermodynamic Temperature", "T", true);
		TIME = new RegularBaseQuantity("Time", "t", true);
		UNKNOWN = new UnknownBaseQuantity();
	}

	/**
	 * Constructs from a name and a symbol.
	 *
	 * @param name
	 *            Name of the base quantity. Shall be neither null
	 *            nor empty.
	 * @param symbol
	 *            Symbol for the base quantity. May be null but
	 *            shall not be empty.
	 * @throws NameException
	 *             name is null or empty.
	 */
	public BaseQuantity(final String name, final String symbol)
			throws NameException {
		this(name, symbol, true);
		if (name == null || name.length() == 0
				|| (symbol != null && symbol.length() == 0)) {
			throw new NameException("Invalid name or symbol");
		}
	}

	/**
	 * Constructs from a name and a symbol. This is the trusted form of the the
	 * constructor for use by subclasses only.
	 *
	 * @param name
	 *            Name of the base quantity. Shall be neither null
	 *            nor empty.
	 * @param symbol
	 *            Symbol for the base quantity. May be null but
	 *            shall not be empty.
	 */
	protected BaseQuantity(final String name, final String symbol,
			final boolean trusted) {
		this.name = name;
		this.symbol = symbol;
	}

	/**
	 * Returns the name of the base quantity.
	 *
	 * @return The name of the base quantity. Shall not be null or
	 *         empty.
	 */
	public String getName() {
		return name;
	}

	/**
	 * Returns the symbol of the base quantity.
	 *
	 * @return The symbol of the base quantity. May be null. If
	 *         non-null, then shall not be empty.
	 */
	public String getSymbol() {
		return symbol;
	}

	/**
	 * Returns the identifier for the base quantity.
	 *
	 * @return The base quantity's identifier (i.e. symbol or name).
	 */
	public final String getID() {
		final String id = getSymbol();
		return id == null
				? getName()
				: id;
	}

	/**
	 * Returns the string representation of the base quantity.
	 *
	 * @return The string representation of the base quantity.
	 */
	@Override
	public final String toString() {
		return getID();
	}

	/**
	 * Indicates if this base quantity is semantically identical to an object.
	 *
	 * @param object
	 *            The object to examine.
	 * @return true if an only if this base quantity is
	 *         semantically identical to object.
	 */
	@Override
	public boolean equals(final Object object) {
		return this == object
				? true
				: !(object instanceof BaseQuantity)
						? false
						: getName().equalsIgnoreCase(
								((BaseQuantity) object).getName())
								&& (getSymbol() == null || getSymbol().equals(
										((BaseQuantity) object).getSymbol()));
	}

	/**
	 * Returns the hash code of this instance.
	 *
	 * @return The hash code of this instance.
	 */
	@Override
	public int hashCode() {
		return getName().toLowerCase().hashCode() ^ (getSymbol() == null
				? 0
				: getSymbol().hashCode());
	}

	/**
	 * Compares this base quantity to another base quantity.
	 *
	 * @param that
	 *            The BaseQuantity to compare against.
	 * @return An integer that is negative, zero, or positive depending on
	 *         whether this BaseQuantity is less than, equal to, or greater than
	 *         that.
	 */
	public int compareTo(final BaseQuantity that) {
		int comp;
		if (this == that) {
			comp = 0;
		}
		else {
			comp = getName().compareToIgnoreCase(that.getName());
			if (comp == 0 && getSymbol() != null) {
				comp = getSymbol().compareTo(that.getSymbol());
			}
		}
		return comp;
	}

	/**
	 * Indicates if this base quantity is dimensionless.
	 *
	 * @return true if an only if this BaseQuantity is
	 *         dimensionless (e.g. BaseQuantity.SOLID_ANGLE).
	 */
	public abstract boolean isDimensionless();

	/**
	 * Tests this class.
	 */
	public static void main(final String[] args) {
		System.out.println("AMOUNT_OF_SUBSTANCE.getName() = "
				+ AMOUNT_OF_SUBSTANCE.getName());
		System.out.println("LUMINOUS_INTENSITY.getSymbol() = "
				+ LUMINOUS_INTENSITY.getSymbol());
		System.out.println("PLANE_ANGLE.getSymbol() = "
				+ PLANE_ANGLE.getSymbol());

		System.out.println("LENGTH.equals(LENGTH) = " + LENGTH.equals(LENGTH));
		System.out.println("LENGTH.equals(MASS) = " + LENGTH.equals(MASS));
		System.out.println("LENGTH.equals(PLANE_ANGLE) = "
				+ LENGTH.equals(PLANE_ANGLE));
		System.out.println("PLANE_ANGLE.equals(PLANE_ANGLE) = "
				+ PLANE_ANGLE.equals(PLANE_ANGLE));
		System.out.println("PLANE_ANGLE.equals(SOLID_ANGLE) = "
				+ PLANE_ANGLE.equals(SOLID_ANGLE));

		System.out.println("LENGTH.compareTo(LENGTH) = "
				+ LENGTH.compareTo(LENGTH));
		System.out
				.println("LENGTH.compareTo(MASS) = " + LENGTH.compareTo(MASS));
		System.out.println("LENGTH.compareTo(PLANE_ANGLE) = "
				+ LENGTH.compareTo(PLANE_ANGLE));
		System.out.println("PLANE_ANGLE.compareTo(PLANE_ANGLE) = "
				+ PLANE_ANGLE.compareTo(PLANE_ANGLE));
		System.out.println("PLANE_ANGLE.compareTo(SOLID_ANGLE) = "
				+ PLANE_ANGLE.compareTo(SOLID_ANGLE));
	}
}


/*
 * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.ui.catalog;

import thredds.client.catalog.*;
import thredds.client.catalog.builder.CatalogBuilder;
import ucar.nc2.ui.widget.BAMutil;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.util.Optional;

import javax.swing.*;
import javax.swing.event.TreeExpansionEvent;
import javax.swing.event.TreeWillExpandListener;
import javax.swing.tree.*;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.beans.PropertyChangeEvent;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;

/**
 * A Swing widget for THREDDS clients to display catalogs in a JTree, and allows
 * the user to select a dataset.
 * When a new catalog is read, or a dataset is selected, a java.beans.PropertyChangeEvent is
 * thrown, see addPropertyChangeListener.
 *

 * Example:
 *

 *  CatalogTreeView  tree = new CatalogTreeView();
    tree.addPropertyChangeListener(  new java.beans.PropertyChangeListener() {
      public void propertyChange( java.beans.PropertyChangeEvent e) {
        if (e.getPropertyName().equals("Selection")) {
          ...
        } else if (e.getPropertyName().equals("Dataset")) {
          ...
        } else if (e.getPropertyName().equals("Catalog")) {
          ...
        }
      }
    });

 *
 *
 * Handles Catalog References internally. Catalogs are read in a background thread, which can be cancelled by the user.
 *
 You probably want to use CatalogChooser or ThreddsDatasetChooser for more complete
 *  functionality.
 * @see CatalogChooser
 * @see ThreddsDatasetChooser
 *
 * @author John Caron
 */

public class CatalogTreeView extends JPanel {
  private Catalog catalog;

    // state
  // private DatasetFilter filter = null;
  private boolean accessOnly = true;
  private String catalogURL = "";
  private boolean openCatalogReferences = true;
  private boolean openDatasetScans = true;

    // ui
  private JTree tree;
  private InvCatalogTreeModel model;

  private boolean debugRef = false;
  private boolean debugTree = false;

  /**
   * Constructor.
   */
  public CatalogTreeView() {
    // the catalog tree
    tree = new JTree(new DefaultTreeModel(new DefaultMutableTreeNode(null, false)));
    tree.setCellRenderer(new MyTreeCellRenderer());

    tree.putClientProperty("JTree.lineStyle", "Angled");
    tree.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
    tree.setToggleClickCount(1);

    tree.addMouseListener( new MouseAdapter() {
      public void mousePressed(MouseEvent e) {
        if (!SwingUtilities.isLeftMouseButton(e)) return; // left button only

        int selRow = tree.getRowForLocation(e.getX(), e.getY());
        if (selRow != -1) {
          checkForCatref();
          InvCatalogTreeNode node = (InvCatalogTreeNode) tree.getLastSelectedPathComponent();
          if (node != null)
            firePropertyChangeEvent(new PropertyChangeEvent(this, "Selection", null, node.ds));
        }

        if ((selRow != -1) && (e.getClickCount() == 2)) {
          acceptSelected();
        }
      }
    });

    tree.addTreeWillExpandListener( new TreeWillExpandListener() {
      public void treeWillCollapse(TreeExpansionEvent evt) { }
      public void treeWillExpand(TreeExpansionEvent evt) {
        InvCatalogTreeNode node = (InvCatalogTreeNode) evt.getPath().getLastPathComponent();
        if (node.ds instanceof CatalogRef) {
          CatalogRef catref = (CatalogRef) node.ds;
          if (!catref.isRead()) {
            if (openCatalogReferences) //  && (openDatasetScans || !isDatasetScan))
              node.readCatref();
          }
        }
      }
    });

    PopupMenu varPopup = new PopupMenu(tree, "Options");
    varPopup.addAction("Open all children", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        InvCatalogTreeNode node = (InvCatalogTreeNode) tree.getLastSelectedPathComponent();
        if (node != null)
          open(node, true);
      }
    });

    varPopup.addAction("Open one level of children", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        InvCatalogTreeNode node = (InvCatalogTreeNode) tree.getLastSelectedPathComponent();
        if (node != null) {
          node.makeChildren( true);
          for (InvCatalogTreeNode child : node.children)
            tree.expandPath(makeTreePath(child));
        }
      }
    });

    ToolTipManager.sharedInstance().registerComponent(tree);

   // layout
    setLayout(new BorderLayout());
    add(new JScrollPane(tree), BorderLayout.CENTER);
  }

  /**
   * Set whether catalog references are opened. default is true.
   */
  public void setOpenCatalogReferences( boolean openCatalogReferences) {
    this.openCatalogReferences = openCatalogReferences;
  }

  /**
   * Set whether catalog references from dataset scans are opened. default is true.
   */
  public void setOpenDatasetScans( boolean openDatasetScans) {
    this.openDatasetScans = openDatasetScans;
  }

  private void firePropertyChangeEvent(DatasetNode ds) {
    PropertyChangeEvent event = new PropertyChangeEvent(this, "Dataset", null, ds);
    firePropertyChangeEvent( event);
  }

  /**
   * Fires a PropertyChangeEvent:
   *

  when a new catalog is read and displayed:
   * propertyName = "Catalog", getNewValue() = catalog URL string
   *
  when a node is selected:
   * propertyName = "Selection", getNewValue() = InvDataset chosen.
   *
  when a node is double-clicked:
   * propertyName = "Dataset", getNewValue() = InvDataset chosen.
   *
  when a TreeNode is added:
   * propertyName = "TreeNode", getNewValue() = InvDataset added.
   *

   */
  private void firePropertyChangeEvent(PropertyChangeEvent event) {
    firePropertyChange(event.getPropertyName(), event.getOldValue(), event.getNewValue());
  }

  /** Whether to throw events only if dataset has an Access.
   *  @param accessOnly if true, throw events only if dataset has an Access
   */
  public void setAccessOnly( boolean accessOnly) { this.accessOnly = accessOnly; }

  /** Get the current catalog. */
  public Catalog getCatalog() { return catalog; }

  /** Get the URL of the current catalog. */
  public String getCatalogURL() { return catalogURL; }
  private void setCatalogURL( String catalogURL) {
    this.catalogURL = catalogURL;
  }

  /**
   * Get the currently selected InvDataset.
   * @return selected InvDataset, or null if none.
   */
  public DatasetNode getSelectedDataset() {
    InvCatalogTreeNode tnode = getSelectedNode();
    return tnode == null ? null : tnode.ds;
  }

  private InvCatalogTreeNode getSelectedNode() {
    Object node = tree.getLastSelectedPathComponent();
    if (node == null) return null;
    if ( !(node instanceof InvCatalogTreeNode)) return null;
    return (InvCatalogTreeNode) node;
  }

  /**
   * Set the currently selected InvDataset.
   * @param ds select this InvDataset, must be already in the tree.
   * LOOK does this work ?? doesnt throw event
   */
  public void setSelectedDataset(Dataset ds) {
    if (ds == null) return;
    TreePath path = makePath(ds);
    if (path == null) return;
    tree.setSelectionPath( path);
    tree.scrollPathToVisible( path);
  }

  /**
   * Create the TreePath corresponding to the InvDataset.
   * @param ds the InvDataset, must be already in the tree.
   * @return the corresponding TreePath.
   */
  TreePath makePath(Dataset ds) {
    return null;
    //TreeNode node = (TreeNode) ds.getUserProperty("TreeNode");  // LOOK
    //return makeTreePath( node);
  }

  /**
   * Create the TreePath corresponding to the given TreeNode.
   * @param node the TreeNode; already in the Tree.
   * @return the corresponding TreePath.
   */
  TreePath makeTreePath(TreeNode node) {
    ArrayList<TreeNode> path = new ArrayList<>();
    path.add( node);
    TreeNode parent = node.getParent();
    while (parent != null) {
      path.add(0, parent);
      parent = parent.getParent();
    }

    Object[] paths = path.toArray();
    return new TreePath(paths);
  }

  /**
   * Open all nodes of the tree.
   * @param includeCatref open catrefs?
   */
  public void openAll( boolean includeCatref) {
    if (catalog == null) return;
    open( (InvCatalogTreeNode) model.getRoot(), includeCatref);
    tree.repaint();
  }

  private void open( InvCatalogTreeNode node, boolean includeCatref) {
    if (node == null) return;
    node.makeChildren( includeCatref);
    tree.expandPath(makeTreePath(node));

    //System.out.printf("open %s%n", node);
    Enumeration e = node.children();
    while (e.hasMoreElements()) {
      InvCatalogTreeNode child = (InvCatalogTreeNode) e.nextElement();
      //System.out.printf(" child %s%n", child);
      open(child, includeCatref);
    }
  }

  void checkForCatref() {
    DatasetNode ds = getSelectedDataset();
    if (ds == null)
      return;

    if ( (ds instanceof CatalogRef)) {
      CatalogRef catref = (CatalogRef) ds;
      //boolean isDatasetScan = catref.isDatasetScan();
      if (!catref.isRead()) {
        if (openCatalogReferences) { //  && (openDatasetScans || !isDatasetScan)) {
          InvCatalogTreeNode tnode = getSelectedNode();
          if (tnode != null)
            tnode.readCatref();
        }
      }
    }
  }

  void acceptSelected() {
    DatasetNode dsn = getSelectedDataset();
    if (dsn == null) return;
    if (accessOnly && dsn instanceof Dataset) {
      Dataset ds = (Dataset) dsn;
      if (!ds.hasAccess()) return;
    }

    //setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
    firePropertyChangeEvent( dsn);
    //setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
  }

  /**
   * Set the InvCatalog to display.
   * The catalog is read asynchronously and displayed if successfully read.
   * You must use a PropertyChangeEventListener to be notified if successful.
   *
   * @param location The URL of the InvCatalog.
   */
  public void setCatalog(String location) {
    CatalogBuilder builder = new CatalogBuilder();

    try {
      Catalog cat = builder.buildFromLocation(location, null);
      setCatalog(cat);

    } catch (Exception ioe) {
      JOptionPane.showMessageDialog(this, "Error opening catalog location " + location+" err="+builder.getErrorMessage());
    }
  }

  public void redisplay() {
    setCatalog( catalog);
  }

  /**
   * Set the catalog to be displayed. If ok, then a "Catalog" PropertyChangeEvent is sent.
   * @param catalog to be displayed
   */
  public void setCatalog(Catalog catalog) {
    if (catalog == null) return;
    String catalogName = catalog.getBaseURI().toString();
    this.catalog = catalog;

    // send catalog event
    setCatalogURL( catalogName);

    // display tree
    // this sends TreeNode events
    model = new InvCatalogTreeModel(catalog);
    tree.setModel( model);

      // debug
    if (debugTree) {
      System.out.println("*** catalog/showJTree =");
      showNode(tree.getModel(), tree.getModel().getRoot());
      System.out.println("*** ");
    }

    // look for a specific dataset
    int pos = catalogName.indexOf('#');
    if (pos >= 0) {
      String id = catalogName.substring( pos+1);
      Dataset dataset = catalog.findDatasetByID( id);
      if (dataset != null) {
        setSelectedDataset(dataset);
        firePropertyChangeEvent( new PropertyChangeEvent(this, "Selection", null, dataset));
      }
    }

    // send catalog event
    firePropertyChangeEvent(new PropertyChangeEvent(this, "Catalog", null, catalogName));
  }

  // debug
  private void showNode(TreeModel tree, Object node) {
    if (node == null) return;
    InvCatalogTreeNode tnode = (InvCatalogTreeNode) node;
    DatasetNode cp = tnode.ds;
    System.out.println(" node= "+cp.getName()+" leaf= "+tree.isLeaf( node));
    for (int i=0; i< tree.getChildCount(node); i++)
      showNode(tree, tree.getChild(node, i));
  }

  // make an InvCatalog into a TreeModel
  private class InvCatalogTreeModel extends javax.swing.tree.DefaultTreeModel {
    InvCatalogTreeModel (DatasetNode top) {
      super( new InvCatalogTreeNode( null, top), false);
    }
  }

  // make an InvDataset into a TreeNode
  // defer opening catalogRefs
  private class InvCatalogTreeNode implements javax.swing.tree.TreeNode {
    DatasetNode ds;
    private InvCatalogTreeNode parent;
    private ArrayList<InvCatalogTreeNode> children = null;
    private boolean isReading = false;

    InvCatalogTreeNode( InvCatalogTreeNode parent, DatasetNode ds) {
      this.parent = parent;
      this.ds = ds;
      // ds.setUserProperty("TreeNode", this);
      if (debugTree) System.out.println("new="+ds.getName()+" ");
      firePropertyChangeEvent(new PropertyChangeEvent(this, "TreeNode", null, ds));
    }

    public Enumeration children() {
      if (debugTree) System.out.println("children="+ds.getName()+" ");
      if (children == null) return Collections.enumeration( new ArrayList<InvCatalogTreeNode>());
      return Collections.enumeration(children);
    }

    public boolean getAllowsChildren() {
      return true;
    }

    public TreeNode getChildAt(int index) {
     if (debugTree) System.out.println("getChildAt="+ds.getName()+" "+index);
     return children.get(index);
    }

    public int getChildCount() {
      if (children == null) makeChildren( false);
      if (children == null) return 0;

      return children.size();
    }

    void makeChildren(boolean force) {
      if (children == null) {
        if (ds instanceof CatalogRef) {
          CatalogRef catref = (CatalogRef) ds;
          if (debugRef) System.out.println("getChildCount on catref="+ds.getName()+" " + catref.isRead()+" "+isReading);
          if (!catref.isRead() && !force) { // dont open it until explicitly asked
            return;
          }
        }

        if (debugRef) System.out.println("getChildCount on ds="+ds.getName()+" ");
        children = new ArrayList<>();
        for (Dataset nested : ds.getDatasets())
          children.add( new InvCatalogTreeNode( this, nested));
      }
    }

    void readCatref() {
      CatalogRef catref = (CatalogRef) ds;
      if (debugRef) System.out.println("readCatref on ="+ds.getName()+" "+isReading);
      if (!isReading) {
        isReading = true;
        try {
          Optional<DatasetNode> opt = catref.readCatref();
          if (!opt.isPresent()) {
            javax.swing.JOptionPane.showMessageDialog(CatalogTreeView.this, opt.getErrorMessage());
            return;
          }
          setCatalog(opt.get());

        } catch (IOException e) {
          javax.swing.JOptionPane.showMessageDialog(CatalogTreeView.this, "Error reading catref " + catref.getName()+" err="+e.getMessage());
        }
      }
    }

    public int getIndex(TreeNode child) {
      if (debugTree) System.out.println("getIndex="+ds.getName()+" "+child);
      return children.indexOf(child);
    }

    public TreeNode getParent() { return parent; }

    public boolean isLeaf() {
      if (debugTree) System.out.println("isLeaf="+ds.getName());
      if (ds instanceof CatalogRef) {
        return false;
      }
      return !ds.hasNestedDatasets();
    }

    public String toString() { return ds.getName(); }

    public void setCatalog(DatasetNode catalog) {
      children = new ArrayList<>();
      java.util.List<Dataset> datasets = catalog.getDatasets();
      if (datasets.size() == 1) {
        Dataset top = datasets.get(0);
        if (top.getName().equalsIgnoreCase(ds.getName())) {
          ds = top; // ??
          datasets = top.getDatasets();
        }
      }
      int[] childIndices = new int[ datasets.size()];
      for (int count = 0; count < datasets.size(); count++) {
        children.add( new InvCatalogTreeNode( this, datasets.get(count)));
        childIndices[count] = count;
      }

      model.nodesWereInserted(this, childIndices);
      // model.nodeStructureChanged( this);
      tree.expandPath( makeTreePath(this));
      if (debugRef) System.out.println("model.nodeStructureChanged on "+this);
      isReading = false;
    }

    public void failed() {
      if (debugRef) System.out.println("failed called on "+this);
      isReading = false;
    }
  }

  // this is to get the inline documentation into a tooltip
  private static class MyTreeCellRenderer
          extends javax.swing.tree.DefaultTreeCellRenderer {
    ImageIcon refIcon, refReadIcon, gridIcon, imageIcon, dqcIcon, dsScanIcon;

    public MyTreeCellRenderer() {
      refIcon = BAMutil.getIcon( "CatalogRef", true);
      refReadIcon = BAMutil.getIcon( "CatalogRefRead", true);
      gridIcon = BAMutil.getIcon( "GridData", true);
      imageIcon = BAMutil.getIcon( "ImageData", true);
      dqcIcon = BAMutil.getIcon( "DQCData", true);
      dsScanIcon = BAMutil.getIcon( "DatasetScan", true);
    }

    public Component getTreeCellRendererComponent(JTree tree, Object value,
      boolean selected, boolean expanded,boolean leaf, int row, boolean hasFocus) {

      Component c = super.getTreeCellRendererComponent(tree, value, selected, expanded, leaf, row, hasFocus);

      if (value instanceof InvCatalogTreeNode) {
        InvCatalogTreeNode node = (InvCatalogTreeNode) value;
        DatasetNode ds = node.ds;

        String doc = ds.toString();
        if (doc != null)
          ((JComponent)c).setToolTipText( doc);

        if (ds instanceof CatalogRef) {
            if (((CatalogRef)ds).isRead()) setIcon(refReadIcon); else setIcon(refIcon);

        } else if (leaf) {
            setIcon( gridIcon);
        }
      }

      return c;
    }

  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import javax.annotation.Nonnull;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.featurecollection.FeatureCollectionConfig.GribConfig;
import thredds.inventory.CollectionUpdateType;
import thredds.inventory.MCollection;
import thredds.inventory.MFile;
import ucar.nc2.grib.GribIndex;
import ucar.nc2.grib.GribIndexCache;
import ucar.nc2.grib.coord.Coordinate;
import ucar.nc2.grib.coord.CoordinateEns;
import ucar.nc2.grib.coord.CoordinateND;
import ucar.nc2.grib.coord.CoordinateRuntime;
import ucar.nc2.grib.coord.CoordinateSharer;
import ucar.nc2.grib.coord.CoordinateTime2D;
import ucar.nc2.grib.coord.CoordinateVert;
import ucar.nc2.grib.coord.GribRecordStats;
import ucar.nc2.grib.grib1.*;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib2.Grib2Utils;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.time.CalendarPeriod;
import ucar.nc2.util.CloseableIterator;

import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Builds indexes for collections of Grib1 files.
 * May create GC or PC
 *
 * @author John
 * @since 2/5/14
 */
public class Grib1CollectionBuilder extends GribCollectionBuilder {
  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Grib1CollectionBuilder.class);

  private final FeatureCollectionConfig.GribConfig gribConfig;
  private Grib1Customizer cust;

  Grib1CollectionBuilder(String name, MCollection dcm, org.slf4j.Logger logger) {
    super(true, name, dcm, logger);

    FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG);
    gribConfig = config.gribConfig;
  }

  // read all records in all files,
  // divide into groups based on GDS hash and optionally the runtime
  // each group has an arraylist of all records that belong to it.
  // for each group, call rectilizer to derive the coordinates and variables
  @Override
  public List<Grib1CollectionWriter.Group> makeGroups(List<MFile> allFiles, boolean singleRuntime, Formatter errlog) throws IOException {
    Map<GroupAndRuntime, Grib1CollectionWriter.Group> gdsMap = new HashMap<>();

    logger.debug("Grib1CollectionBuilder {}: makeGroups", name);
    int fileno = 0;
    GribRecordStats statsAll = new GribRecordStats(); // debugging

    logger.debug(" dcm={}", dcm);

    // place each record into its group
    try (CloseableIterator<MFile> iter = dcm.getFileIterator()) { // not sorted
      if (iter == null)
        return new ArrayList<>(); // empty

      while (iter.hasNext()) {
        MFile mfile = iter.next();
        Grib1Index index;
        try {
          if (Grib.debugGbxIndexOnly) {
            index = (Grib1Index) GribIndex.open(true, mfile);
            if (index == null) continue;
          } else {
            // here is where gbx9 files get recreated
            index = (Grib1Index) GribIndex.readOrCreateIndexFromSingleFile(true, mfile, CollectionUpdateType.test, logger);
          }
          allFiles.add(mfile);  // add on success

        } catch (IOException ioe) {
          logger.error("Grib1CollectionBuilder " + name + " : reading/Creating gbx9 index for file " + mfile.getPath() + " failed", ioe);
          continue;
        }
        if (index == null) {
          logger.error("Grib1CollectionBuilder " + name + " : reading/Creating gbx9 index for file " + mfile.getPath() + " failed");
          continue;
        }

        for (Grib1Record gr : index.getRecords()) { // we are using entire Grib1Record - likely this is the memory bottleneck for how big a collection can handle
          if (this.cust == null) {
            cust = Grib1Customizer.factory(gr, null);
            cust.setTimeUnitConverter(gribConfig.getTimeUnitConverter());
          }
          if (filterIntervals(gr, gribConfig.intvFilter)) {
            statsAll.filter++;
            continue; // skip
          }

          gr.setFile(fileno); // each record tracks which file it belongs to
          Grib1Gds gds = gr.getGDS();  // use GDS to group records
          int hashCode = gribConfig.convertGdsHash(gds.hashCode());  // allow external config to muck with gdsHash. Why? because of error in encoding and we need exact hash matching
          if (0 == hashCode) continue; // skip this group
          //GdsHashObject gdsHashObject = new GdsHashObject(gr.getGDS(), hashOverride);

          CalendarDate runtimeDate = gr.getReferenceDate();
          long runtime = singleRuntime ? runtimeDate.getMillis() : 0;  // seperate Groups for each runtime, if singleRuntime is true
          GroupAndRuntime gar = new GroupAndRuntime(hashCode, runtime);
          Grib1CollectionWriter.Group g = gdsMap.get(gar);
          if (g == null) {
            g = new Grib1CollectionWriter.Group(gr.getGDSsection(), hashCode, runtimeDate);
            gdsMap.put(gar, g);
          }
          g.records.add(gr);
          g.runtimes.add(runtimeDate.getMillis());
        }
        fileno++;
        statsAll.recordsTotal += index.getRecords().size();
      }
    }

    // rectilyze each group independently
    List<Grib1CollectionWriter.Group> groups = new ArrayList<>(gdsMap.values());
    for (Grib1CollectionWriter.Group g : groups) {
      GribRecordStats stats = new GribRecordStats(); // debugging
      Grib1Rectilyser rect = new Grib1Rectilyser(g.records, g.hashCode);
      rect.make(gribConfig, stats, errlog);
      g.gribVars = rect.gribvars;
      g.coords = rect.coords;

      statsAll.add(stats);
    }

    // debugging and validation
    if (logger.isDebugEnabled()) logger.debug(statsAll.show());

    return groups;
  }

      // true means remove
  private boolean filterIntervals(Grib1Record gr, FeatureCollectionConfig.GribIntvFilter intvFilter) {
    Grib1SectionProductDefinition pdss = gr.getPDSsection();
    Grib1ParamTime ptime = gr.getParamTime(cust);
    if (!ptime.isInterval()) return false;

    int[] intv = ptime.getInterval();
    if (intv == null) return false;
    int haveLength = intv[1] - intv[0];

    // keep zero length intervals unless configured otherwise
    if (haveLength == 0 && intvFilter != null && intvFilter.isZeroExcluded())
      return true;

    if (intvFilter != null && intvFilter.hasFilter()) {
      int center = pdss.getCenter();
      int subcenter = pdss.getSubCenter();
      int version = pdss.getTableVersion();
      int param = pdss.getParameterNumber();
      int id = (center << 8) + (subcenter << 16) + (version << 24) + param;

      return intvFilter.filter(id, intv[0], intv[1], Integer.MIN_VALUE);
    }

    return false;
  }

  @Override
  protected boolean writeIndex(String name, String indexFilepath, CoordinateRuntime masterRuntime,
                   List<? extends GribCollectionBuilder.Group> groups, List<MFile> files, CalendarDateRange dateRange) throws IOException {
    Grib1CollectionWriter writer = new Grib1CollectionWriter(dcm, logger);
    List<Grib1CollectionWriter.Group> groups2 = new ArrayList<>();
    for (Object g : groups) groups2.add((Grib1CollectionWriter.Group) g);  // why copy ?
    File indexFileInCache = GribIndexCache.getFileOrCache(indexFilepath);
    return writer.writeIndex(name, indexFileInCache, masterRuntime, groups2, files, type, dateRange);
  }

  public static class VariableBag implements Comparable<VariableBag> {
    final Grib1Record first;
    final Grib1Variable gv;

    final List<Grib1Record> atomList = new ArrayList<>(100); // not sorted
    CoordinateND<Grib1Record> coordND;
    CalendarPeriod timeUnit;

    List<Integer> coordIndex; // index into List<Coordinate>
    long pos;
    int length;

    private VariableBag(Grib1Record first, Grib1Variable gv) {
      this.first = first;
      this.gv = gv;
    }

    @Override
    public int compareTo(@Nonnull VariableBag o) {
      return Grib1Utils.extractParameterCode(first).compareTo(Grib1Utils.extractParameterCode(o.first));
    }

    @Override
    public String toString() {
      return "VariableBag{" +
              ", variable=" + gv.makeVariableName(new GribConfig()) +
              ", coordND=" + coordND +
              ", timeUnit=" + timeUnit +
              ", coordIndex=" + coordIndex +
              ", pos=" + pos +
              ", length=" + length +
              '}';
    }
  }

  // for a single group, create multidimensional (rectangular) variables
  private class Grib1Rectilyser {
    private final int hashCode;
    private final List<Grib1Record> records;
    private List<VariableBag> gribvars;
    private List<Coordinate> coords;

    Grib1Rectilyser(List<Grib1Record> records, int hashCode) {
      this.records = records;
      this.hashCode = hashCode;
      //int gdsHash = gribConfig.convertGdsHash(gdsHashObject.hashCode());
      //gdsHashOverride = (gdsHash == gdsHashObject.hashCode()) ? 0 : gdsHash;
    }

    public void make(FeatureCollectionConfig.GribConfig config, GribRecordStats counter, Formatter info) {
      CalendarPeriod userTimeUnit = config.userTimeUnit;

      // assign each record to unique variable using cdmVariableHash()
      Map<Grib1Variable, VariableBag> vbHash = new HashMap<>(100);
      for (Grib1Record gr : records) {
        Grib1Variable cdmHash;
        try {
          cdmHash =  new Grib1Variable(cust, gr, hashCode, gribConfig.useTableVersion, gribConfig.intvMerge, gribConfig.useCenter);
        } catch (Throwable t) {
          logger.warn("Exception on record ", t);
          continue; // keep going
        }
        VariableBag bag = vbHash.get(cdmHash);
        if (bag == null) {
          bag = new VariableBag(gr, cdmHash);
          vbHash.put(cdmHash, bag);
        }
        bag.atomList.add(gr);
      }
      gribvars = new ArrayList<>(vbHash.values());
      Collections.sort(gribvars); // make it deterministic by sorting

      // create dense coordinates for each variable
      for (VariableBag vb : gribvars) {
        Grib1SectionProductDefinition pdss = vb.first.getPDSsection();
        Grib1ParamTime ptime = vb.first.getParamTime(cust);

        int unit = cust.convertTimeUnit(pdss.getTimeUnit());
        vb.timeUnit = userTimeUnit == null ? Grib2Utils.getCalendarPeriod(unit) : userTimeUnit; // so can override the code // ok for GRIB1
        CoordinateND.Builder<Grib1Record> coordNBuilder = new CoordinateND.Builder<>();

        boolean isTimeInterval = ptime.isInterval();
        /* if (isDense) { // time is runtime X time coord
          coordNBuilder.addBuilder(new CoordinateRuntime.Builder1(vb.timeUnit));
          if (isTimeInterval)
            coordNBuilder.addBuilder(new CoordinateTimeIntv.Builder1(cust, unit, vb.timeUnit, null)); // null refdate not ok
          else
            coordNBuilder.addBuilder(new CoordinateTime.Builder1(cust, pdss.getTimeUnit(), vb.timeUnit, null)); // null refdate not ok

        } else {  */
          // time is kept as 2D coordinate, separate list of times for each runtime
        CoordinateTime2D.Builder1 builder2D = new CoordinateTime2D.Builder1(isTimeInterval, cust, vb.timeUnit, unit);
        coordNBuilder.addBuilder(builder2D);
        //}

        if (vb.first.getPDSsection().isEnsemble())
          coordNBuilder.addBuilder(new CoordinateEns.Builder1(cust, 0));

        if (cust.isVerticalCoordinate(pdss.getLevelType()))
          coordNBuilder.addBuilder(new CoordinateVert.Builder1(cust, pdss.getLevelType()));

        // populate the coordinates with the inventory of data
        for (Grib1Record gr : vb.atomList)
          coordNBuilder.addRecord(gr);

        // done, build coordinates and sparse array indicating which records to use
        vb.coordND = coordNBuilder.finish(vb.atomList, info);
      }

      // make shared coordinates across variables
      CoordinateSharer<Grib1Record> sharify = new CoordinateSharer<>(config.unionRuntimeCoord, logger);
      for (VariableBag vb : gribvars) {
        sharify.addCoords(vb.coordND.getCoordinates());
      }
      sharify.finish();
      this.coords = sharify.getUnionCoords();

      int tot_used = 0;
      int tot_dups = 0;
      int total = 0;

      // redo the variables against the shared coordinates
      for (VariableBag vb : gribvars) {
        vb.coordND = sharify.reindexCoordND(vb.coordND);
        vb.coordIndex = sharify.reindex2shared(vb.coordND.getCoordinates());
        tot_used += vb.coordND.getSparseArray().countNotMissing();
        tot_dups += vb.coordND.getSparseArray().getNdups();
        total += vb.coordND.getSparseArray().getTotalSize();
       }

      // track stats
      counter.recordsUnique += tot_used;
      counter.dups += tot_dups;
      counter.vars += gribvars.size();
      counter.recordsTotal += total;
    }

    // debugging only
    public void showInfo(Formatter f, Grib1Customizer cust1) {
      //f.format("%nVariables%n");
      //f.format("%n  %3s %3s %3s%n", "time", "vert", "ens");
      GribRecordStats all = new GribRecordStats();

      for (VariableBag vb : gribvars) {
        f.format("Variable %s (%d)%n", Grib1Iosp.makeVariableName(cust, gribConfig, vb.first.getPDSsection()), vb.hashCode());
        vb.coordND.showInfo(f, all);
        //f.format("  %3d %3d %3d %s records = %d density = %f hash=%d", vb.timeCoordIndex, vb.vertCoordIndex, vb.ensCoordIndex,
        //        vname, vb.atomList.size(), vb.recordMap.density(), vb.cdmHash);
        f.format("%n");
      }
      f.format("%n all= %s", all.show());
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.dataset.conv;

import ucar.nc2.*;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.constants.AxisType;
import ucar.nc2.util.CancelTask;
import ucar.nc2.dataset.*;
import ucar.unidata.geoloc.*;
import ucar.unidata.geoloc.projection.*;

import java.util.*;

/**
 * GDV Conventions.
 * Deprecated - use CF or _Coordinates.
 * DefaultConvention is now the default Convention
 * @author caron
 */

public class GDVConvention extends CSMConvention {
  protected ProjectionCT projCT = null;

  public GDVConvention() {
    this.conventionName = "GDV";
    checkForMeter = false;
  }

  public void augmentDataset(NetcdfDataset ds, CancelTask cancelTask) {

    projCT = makeProjectionCT(ds);
    if (projCT != null) {
      VariableDS v = makeCoordinateTransformVariable(ds, projCT);
      ds.addVariable(null, v);

      String xname = findCoordinateName(ds, AxisType.GeoX);
      String yname = findCoordinateName(ds, AxisType.GeoY);
      if (xname != null && yname != null)
        v.addAttribute(new Attribute(_Coordinate.Axes, xname + " " + yname));
    }

    ds.finish();
  }

  /**
   * look for aliases.
   */
  @Override
  protected void findCoordinateAxes(NetcdfDataset ds) {

    for (VarProcess vp : varList) {
      if (vp.isCoordinateVariable) continue;

      Variable ncvar = vp.v;
      if (!(ncvar instanceof VariableDS)) continue; // cant be a structure

      String dimName = findAlias(ds, ncvar);
      if (dimName.length() == 0) // none
        continue;
      Dimension dim = ds.findDimension(dimName);
      if (null != dim) {
        vp.isCoordinateAxis = true;
        parseInfo.format(" Coordinate Axis added (GDV alias) = %s for dimension %s%n", vp.v.getFullName(), dimName);
      }
    }

    super.findCoordinateAxes(ds);

    // desperado
    findCoordinateAxesForce(ds);
  }

  private void findCoordinateAxesForce(NetcdfDataset ds) {

    HashMap<AxisType, VarProcess> map = new HashMap<>();

    // find existing axes, so we dont duplicate
    for (VarProcess vp : varList) {
      if (vp.isCoordinateAxis) {
        AxisType atype = getAxisType(ds, (VariableEnhanced) vp.v);
        if (atype != null)
          map.put(atype, vp);
      }
    }

    // look for variables to turn into axes
    for (VarProcess vp : varList) {
      if (vp.isCoordinateVariable) continue;
      Variable ncvar = vp.v;
      if (!(ncvar instanceof VariableDS)) continue; // cant be a structure

      AxisType atype = getAxisType(ds, (VariableEnhanced) vp.v);
      if (atype != null) {
        if (map.get(atype) == null) {
          vp.isCoordinateAxis = true;
          parseInfo.format(" Coordinate Axis added (GDV forced) = %s  for axis %s%n", vp.v.getFullName(), atype);
        }
      }
    }
  }

  /**
   * look for aliases.
   *
   * @param ds       containing dataset
   * @param axisType look for this axis type
   * @return name of axis of that type
   */
  private String findCoordinateName(NetcdfDataset ds, AxisType axisType) {

    List<Variable> vlist = ds.getVariables();
    for (Variable aVlist : vlist) {
      VariableEnhanced ve = (VariableEnhanced) aVlist;
      if (axisType == getAxisType(ds, ve)) {
        return ve.getFullName();
      }
    }
    return null;
  }

  protected void makeCoordinateTransforms(NetcdfDataset ds) {
    if (projCT != null) {
      VarProcess vp = findVarProcess(projCT.getName(), null);
      if (vp != null)
        vp.ct = projCT;
    }
    super.makeCoordinateTransforms(ds);
  }

  protected AxisType getAxisType(NetcdfDataset ds, VariableEnhanced ve) {

    Variable v = (Variable) ve;
    String vname = v.getShortName();

    if (vname.equalsIgnoreCase("x") || findAlias(ds, v).equalsIgnoreCase("x"))
      return AxisType.GeoX;

    if (vname.equalsIgnoreCase("lon") || vname.equalsIgnoreCase("longitude") || findAlias(ds, v).equalsIgnoreCase("lon"))
      return AxisType.Lon;

    if (vname.equalsIgnoreCase("y") || findAlias(ds, v).equalsIgnoreCase("y"))
      return AxisType.GeoY;

    if (vname.equalsIgnoreCase("lat") || vname.equalsIgnoreCase("latitude") || findAlias(ds, v).equalsIgnoreCase("lat"))
      return AxisType.Lat;

    if (vname.equalsIgnoreCase("lev") || findAlias(ds, v).equalsIgnoreCase("lev") ||
            (vname.equalsIgnoreCase("level") || findAlias(ds, v).equalsIgnoreCase("level")))
      return AxisType.GeoZ;

    if (vname.equalsIgnoreCase("z") || findAlias(ds, v).equalsIgnoreCase("z") ||
            (vname.equalsIgnoreCase("altitude") || vname.equalsIgnoreCase("depth")))
      return AxisType.Height;

    if (vname.equalsIgnoreCase("time") || findAlias(ds, v).equalsIgnoreCase("time"))
      return AxisType.Time;

     return super.getAxisType(ds, ve);
  }

  // look for an coord_axis or coord_alias attribute
  private String findAlias(NetcdfDataset ds, Variable v) {
    String alias = ds.findAttValueIgnoreCase(v, "coord_axis", null);
    if (alias == null)
      alias = ds.findAttValueIgnoreCase(v, "coord_alias", "");
    return alias;
  }

  private ProjectionCT makeProjectionCT(NetcdfDataset ds) {
    // look for projection in global attribute
    String projection = ds.findAttValueIgnoreCase(null, "projection", null);
    if (null == projection) {
      parseInfo.format("GDV Conventions error: NO projection name found %n");
      return null;
    }
    String params = ds.findAttValueIgnoreCase(null, "projection_params", null);
    if (null == params) params = ds.findAttValueIgnoreCase(null, "proj_params", null);
    if (null == params) {
      parseInfo.format("GDV Conventions error: NO projection parameters found %n");
      return null;
    }

    // parse the parameters
    int count = 0;
    double[] p = new double[4];
    try {
      // new way : just the parameters
      StringTokenizer stoke = new StringTokenizer(params, " ,");
      while (stoke.hasMoreTokens() && (count < 4)) {
        p[count++] = Double.parseDouble(stoke.nextToken());
      }
    } catch (NumberFormatException e) {
      // old way : every other one
      StringTokenizer stoke = new StringTokenizer(params, " ,");
      while (stoke.hasMoreTokens() && (count < 4)) {
        stoke.nextToken(); // skip
        p[count++] = Double.parseDouble(stoke.nextToken());
      }

    }

    parseInfo.format("GDV Conventions projection %s params = %f %f %f %f%n", projection, p[0],p[1],p[2],p[3]);

    ProjectionImpl proj;
    if (projection.equalsIgnoreCase("LambertConformal"))
      proj = new LambertConformal(p[0], p[1], p[2], p[3]);
    else if (projection.equalsIgnoreCase("TransverseMercator"))
      proj = new TransverseMercator(p[0], p[1], p[2]);
    else if (projection.equalsIgnoreCase("Stereographic") || projection.equalsIgnoreCase("Oblique_Stereographic"))
      proj = new Stereographic(p[0], p[1], p[2]);
    else {
      parseInfo.format("GDV Conventions error: Unknown projection %s%n", projection);
      return null;
    }

    return new ProjectionCT(proj.getClassName(), "FGDC", proj);
  }

}