/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.catalog;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.time.CalendarDuration;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.unidata.util.Format;
import ucar.unidata.util.StringUtil2;

import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;

/**
 * Concrete implementation of a thredds Dataset, for reading and writing from XML.
 *
 * @author john caron
 * @see InvDataset
 */

/*
  Notes on inheritence implementation.
  Version 1.0
    "Local metadata" is the metadata contained in the dataset node itself.
    Inheritable metadata must be placed in an InvMetadata object with inherit = true;

    "Inherited metadata" is not local, but comes from an ancestor node.
    "Public metadata" is all the dataset's metadata, the local and the inherited.
    When finish() is called, the public metadata is constructed.

    If metadata cannot be inherited, it can be safely stored in the InvDataset objec. Otherwise, it must be kept in
    local or inheritable ThreddsMetadata.
*/
public class InvDatasetImpl extends InvDataset {
  static private final Logger logger = LoggerFactory.getLogger(InvDatasetImpl.class);

  private String urlPath;
  private String alias;
  private double size = 0.0;

  private List<InvAccess> accessLocal = new ArrayList<>();
  private List<InvService> servicesLocal = new ArrayList<>();
  protected ThreddsMetadata tm = new ThreddsMetadata(false); // all local metadata kept here. This may include inheritable InvMetadata
  protected ThreddsMetadata tmi = new ThreddsMetadata(true); // local inheritable metadata (canonicalization)
  protected org.jdom2.Element ncmlElement;

  // validation
  protected StringBuilder log = new StringBuilder();
  // filter
  protected boolean mark = false;

  /**
   * Constructor from Catalog XML info. You must call finish() before this object
   * is ready to be used. We had to do it this way so that nested service elements could be added
   * through addService(), before we define on the default service.
   *
   * @param parent      : parent dataset
   * @param name        : display name of dataset
   * @param dataType    : DataType name (may be null)
   * @param serviceName : default service (may be null)
   * @param urlPath     : URL = server.getURLbase() + urlPath
   */
  public InvDatasetImpl(InvDatasetImpl parent, String name, FeatureType dataType, String serviceName, String urlPath) {
    super(parent, name);

    tm.setDataType(dataType);
    tm.setServiceName(serviceName);
    this.urlPath = urlPath;
  }

  /**
   * Finish constructing after all elements have been added.
   * This does the inheritence thing
   * This can be called again if new elements are added.
   *
   * @return true if successful.
   */
  public boolean finish() {
    boolean ok = true;
    java.util.Iterator iter;
    logger.debug("Now finish " + getName() + " id= " + getID());

    authorityName = null;
    dataType = null;
    dataFormatType = null;
    defaultService = null;

    gc = null;
    tc = null;
    docs = new ArrayList<>();
    metadata = new ArrayList<>();
    properties = new ArrayList<>();

    creators = new ArrayList<>();
    contributors = new ArrayList<>();
    dates = new ArrayList<>();
    keywords = new ArrayList<>();
    projects = new ArrayList<>();
    publishers = new ArrayList<>();
    variables = new ArrayList<>();

    canonicalize(); // canonicalize thredds metadata
    transfer2PublicMetadata(tm, true); // add local metadata
    transfer2PublicMetadata(tmi, true); // add local inherited metadata
    transferInheritable2PublicMetadata((InvDatasetImpl) getParent()); // add inheritable metadata from parents

    // build the expanded access list
    access = new ArrayList<>();

    // add access element if urlPath is specified
    if ((urlPath != null) && (getServiceDefault() != null)) {
      InvAccessImpl a = new InvAccessImpl(this, urlPath, getServiceDefault());
      a.setSize(size);
      a.finish();
      addExpandedAccess(a);
    }

    // add local access elements
    iter = accessLocal.iterator();
    while (iter.hasNext()) {
      InvAccessImpl a = (InvAccessImpl) iter.next();
      a.finish();
      addExpandedAccess(a);
    }

    // recurse into child datasets.
    if (!(this instanceof InvCatalogRef)) {
      for (InvDataset invDataset : this.getDatasets()) {
        InvDatasetImpl curDs = (InvDatasetImpl) invDataset;
        ok &= curDs.finish();
      }
    }

    return ok;
  }

  /**
   * Look for InvMetadata elements in the parent that need to be added to the public metadata of this dataset.
   * Recurse up through all ancestors.
   *
   * @param parent transfer from here
   */
  private void transferInheritable2PublicMetadata(InvDatasetImpl parent) {
    if (parent == null) return;
    logger.debug(" inheritFromParent= " + parent.getID());

    transfer2PublicMetadata(parent.getLocalMetadataInheritable(), true);
    //transfer2PublicMetadata(parent.getCat6Metadata(), true);

    /* look through local metadata, find inherited InvMetadata elements
    ThreddsMetadata tmd = parent.getLocalMetadata();
    Iterator iter = tmd.getMetadata().iterator();
    while (iter.hasNext()) {
      InvMetadata meta = (InvMetadata) iter.next();
      if (meta.isInherited()) {
        if (!meta.isThreddsMetadata()) {
          metadata.add(meta);
        } else {
          if (debugInherit) System.out.println("  inheritMetadata Element " + tmd.isInherited() + " " + meta.isInherited());
          meta.finish(); // make sure XLink is read in.
          transfer2PublicMetadata(meta.getThreddsMetadata(), false);
        }
      }
    } */

    // recurse
    transferInheritable2PublicMetadata((InvDatasetImpl) parent.getParent());
  }

  /**
   * take all elements from tmd and add to the public metadata of this dataset.
   * for InvMetadata elements, only add if inheritAll || InvMetadata.isInherited().
   *
   * @param tmd        tahe metadata from here
   * @param inheritAll true if all should be inherited, else only those which are specifically mareked
   */
  private void transfer2PublicMetadata(ThreddsMetadata tmd, boolean inheritAll) {
    if (tmd == null) return;

    logger.debug("  transferMetadata " + tmd);

    if (authorityName == null)
      authorityName = tmd.getAuthority();
    if (dataType == null || (dataType == FeatureType.ANY))
      dataType = tmd.getDataType();
    if (dataFormatType == null || dataFormatType == DataFormatType.NONE)
      dataFormatType = tmd.getDataFormatType();

    if (defaultService == null)
      defaultService = findService(tmd.getServiceName());

    if (gc == null) {
      ThreddsMetadata.GeospatialCoverage tgc = tmd.getGeospatialCoverage();
      if ((tgc != null) && !tgc.isEmpty())
        gc = tgc;
    }

    if (tc == null) {
      DateRange ttc = tmd.getTimeCoverage();
      if (ttc != null) {
        tc = ttc;
      }
    }

    if (tc == null)
      tc = tmd.getTimeCoverage();

    for (InvProperty item : tmd.getProperties()) {
      logger.debug("  add Property " + item + " to " + getID());
      properties.add(item);
    }

    creators.addAll(tmd.getCreators());
    contributors.addAll(tmd.getContributors());
    dates.addAll(tmd.getDates());
    docs.addAll(tmd.getDocumentation());
    keywords.addAll(tmd.getKeywords());
    projects.addAll(tmd.getProjects());
    publishers.addAll(tmd.getPublishers());
    variables.addAll(tmd.getVariables());
    if (variableMapLink == null)
      variableMapLink = tmd.variableMapLink;

    for (InvMetadata meta : tmd.getMetadata()) {
      if (meta.isInherited() || inheritAll) {
        if (!meta.isThreddsMetadata()) {
          metadata.add(meta);
        } else {
          logger.debug("  add metadata Element " + tmd.isInherited() + " " + meta);
          meta.finish(); // make sure XLink is read in.
          transfer2PublicMetadata(meta.getThreddsMetadata(), inheritAll);
          metadata.add(meta);
        }
      }
    }

  }

  /**
   * Transfer all inheritable metadata from fromDs to the local metadata of this dataset.
   * Called by InvDatasetScan to transfer inheritable metaddata to the nested catalogRef
   *
   * @param fromDs transfer from here
   */
  public void transferMetadata(InvDatasetImpl fromDs, boolean copyInheritedMetadataFromParents) {
    if (fromDs == null) return;
    logger.debug(" transferMetadata= " + fromDs.getName());

    if (this != fromDs)
      getLocalMetadata().add(fromDs.getLocalMetadata(), false);

    transferInheritableMetadata(fromDs, getLocalMetadataInheritable(), copyInheritedMetadataFromParents);

    setResourceControl(fromDs.getRestrictAccess());
  }

  /**
   * transfer inherited metadata, consolidating it into target
   *
   * @param fromDs transfer from here, plus its parents
   * @param target transfer to here
   */
  private void transferInheritableMetadata(InvDatasetImpl fromDs, ThreddsMetadata target,
                                           boolean copyInheritedMetadataFromParents) {
    if (fromDs == null) return;
    logger.debug(" transferInheritedMetadata= " + fromDs.getName());

    target.add(fromDs.getLocalMetadataInheritable(), true);

    /* look through local metadata, find inherited InvMetadata elements
    ThreddsMetadata tmd = fromDs.getLocalMetadata();
    Iterator iter = tmd.getMetadata().iterator();
    while (iter.hasNext()) {
      InvMetadata meta = (InvMetadata) iter.next();
      if (meta.isInherited()) {
        if (!meta.isThreddsMetadata()) {
          tmc.addMetadata( meta);
        } else {
          logger.debug("  transferInheritedMetadata "+meta.hashCode()+" = "+meta);
          meta.finish(); // LOOK ?? make sure XLink is read in.
          tmc.add( meta.getThreddsMetadata(), true);
        }
      }
    }   */

    // now do the same for the parents
    if (copyInheritedMetadataFromParents)
      transferInheritableMetadata((InvDatasetImpl) fromDs.getParent(), target, true);
  }

  private void addExpandedAccess(InvAccessImpl a) {
    InvService service = a.getService();
    if (null == service) {
      a.check(log, false); // illegal; get error message
      return;
    }

    if (service.getServiceType() == ServiceType.COMPOUND) {
      // if its a compound service, expand it
      for (InvService nestedService : service.getServices()) {
        InvAccessImpl nestedAccess = new InvAccessImpl(this, a.getUrlPath(), nestedService);
        addExpandedAccess(nestedAccess); // i guess it could recurse
      }
    } else {
      access.add(a);
    }
  }

  /**
   * Put metadata into canonical form.
   * All non-inherited thredds metadata put into single metadata element, pointed to by getLocalMetadata().
   * All inherited thredds metadata put into single metadata element, pointed to by getLocalMetadataInherited().
   * This is needed to do reliable editing.
   */
  protected void canonicalize() {
    List<InvMetadata> whatsLeft = new ArrayList<>();
    List<InvMetadata> original = new ArrayList<>(tm.metadata); // get copy of metadata
    tm.metadata = new ArrayList<>();

    // transfer all non-inherited thredds metadata to tm
    // transfer all inherited thredds metadata to tmi
    for (InvMetadata m : original) {
      if (m.isThreddsMetadata() && !m.isInherited() && !m.hasXlink()) {
        ThreddsMetadata nested = m.getThreddsMetadata();
        tm.add(nested, false);

      } else if (m.isThreddsMetadata() && m.isInherited() && !m.hasXlink()) {
        ThreddsMetadata nested = m.getThreddsMetadata();
        tmi.add(nested, true);

      } else {
        whatsLeft.add(m);
      }
    }

    // non ThreddsMetadata goes into tm
    tm.metadata.addAll(whatsLeft);
  }

  /**
   * Construct an InvDatasetImpl which refers to a urlPath.
   * This is used to create a standalone InvDatasetImpl, outside of an InvCatalog.
   * An "anonymous" InvServerImpl is created and attached to the InvDataset.
   *
   * @param urlPath  : construct URL from this path
   * @param dataType : data type
   * @param stype    : ServiceType
   */
  public InvDatasetImpl(String urlPath, FeatureType dataType, ServiceType stype) {
    super(null, "local file");
    tm.setDataType(dataType);
    tm.setServiceName("anon");
    this.urlPath = urlPath;

    // create anonymous service
    addService(new InvService(tm.getServiceName(), stype.toString(), "", "", null));

    finish();
  }

  public InvDatasetImpl(InvDataset parent, String name) {
    super(parent, name);
  }

  /**
   * copy constructor
   *
   * @param from copy from here
   */
  public InvDatasetImpl(InvDatasetImpl from) {
    super(from.getParent(), from.getName());

    // steal everything
    this.tm = new ThreddsMetadata(from.getLocalMetadata());
    this.tmi = new ThreddsMetadata(from.getLocalMetadataInheritable());
    this.accessLocal = new ArrayList<>(from.getAccessLocal());
    this.servicesLocal = new ArrayList<>(from.getServicesLocal());

    this.harvest = from.harvest;
    this.collectionType = from.collectionType;
  }

  ////////////////////////////////////////////////////////
  // get/set local properties

  /**
   * @return alias for this Dataset, if there is one
   */
  public String getAlias() {
    return alias;
  }

  /**
   * Set alias for this Dataset
   *
   * @param alias ID of another Dataset
   */
  public void setAlias(String alias) {
    this.alias = alias;
    hashCode = 0;
  }

  /**
   * Set the containing catalog; use only for top level dataset.
   *
   * @param catalog the containing catalog for the top level dataset.
   */
  public void setCatalog(InvCatalog catalog) {
    this.catalog = catalog;
    hashCode = 0;
  }

  /**
   * Get real parent dataset, no proxies
   *
   * @return parent dataset. If top dataset, return null.
   */
  public InvDataset getParentReal() {
    return parent;
  }

  /**
   * Get urlPath for this Dataset
   *
   * @return urlPath for this Dataset
   */
  public String getUrlPath() {
    return urlPath;
  }

  /**
   * Set the urlPath for this InvDatasetImpl
   *
   * @param urlPath the urlPath for this InvDatasetImpl
   */
  public void setUrlPath(String urlPath) {
    this.urlPath = urlPath;
    hashCode = 0;
  }

  /**
   * Set authorityName for this Dataset
   *
   * @param authorityName for this Dataset
   */
  public void setAuthority(String authorityName) {
    tm.setAuthority(authorityName);
    hashCode = 0;
  }

  ////////////////////////////////////////////////////////
  // setters for public properties (in InvDataset)

  /**
   * Set collectionType
   *
   * @param collectionType the collection type
   */
  public void setCollectionType(CollectionType collectionType) {
    this.collectionType = collectionType;
    hashCode = 0;
  }

  /**
   * Set harvest
   *
   * @param harvest true if this dataset should be harvested for Digital Libraries
   */
  public void setHarvest(boolean harvest) {
    this.harvest = harvest;
    hashCode = 0;
  }

  /**
   * Set the ID for this Dataset
   *
   * @param id unique ID
   */
  public void setID(String id) {
    this.id = id;
    hashCode = 0;
  }

  /**
   * Set name of this Dataset.
   *
   * @param name of the dataset
   */
  public void setName(String name) {
    this.name = name;
    hashCode = 0;
  }

  /**
   * Set the parent dataset.
   *
   * @param parent parent dataset
   */
  public void setParent(InvDatasetImpl parent) {
    this.parent = parent;
    hashCode = 0;
  }

  ////////////////////////////////////////////////////////
  // setters for public properties - these go into the local metadata object
  // these are not inherited, only InvMetadata objects are inheritable.

  // LOOK these are probably wrong

  public void setGeospatialCoverage(ThreddsMetadata.GeospatialCoverage gc) {
    tm.setGeospatialCoverage(gc);
    hashCode = 0;
  }

  public void setTimeCoverage(CalendarDateRange tc) {
    tm.setTimeCoverage(tc);
    hashCode = 0;
  }

  public void setTimeCoverage(DateRange tc) {
    tm.setTimeCoverage(tc);
    hashCode = 0;
  }

  public void setDataFormatType(DataFormatType dataFormatType) {
    tm.setDataFormatType(dataFormatType);
    hashCode = 0;
  }

  public void setDataType(FeatureType dataType) {
    tm.setDataType(dataType);
    hashCode = 0;
  }

  public double getDataSize() {
    return tm.getDataSize();
  }

  public void setDataSize(double dataSize) {
    tm.setDataSize(dataSize);
    hashCode = 0;
  }

  public DateType getLastModifiedDate() {
    // Look for a last modified date.
    for (DateType dateType : tm.getDates()) {
      if ((dateType.getType() != null) && dateType.getType().equals("modified")) {
        return dateType;
      }
    }
    return null;
  }

  public void setLastModifiedDate(DateType lastModDate) {
    if (lastModDate == null)
      throw new IllegalArgumentException("Last modified date can't be null.");
    if (lastModDate.getType() == null || !lastModDate.getType().equals("modified")) {
      throw new IllegalArgumentException("Date type must be \"modified\" (was \"" + lastModDate.getType() + "\").");
    }

    // Check for existing last modified date and remove if one exists.
    DateType curLastModDateType = this.getLastModifiedDate();
    if (curLastModDateType != null) {
      tm.getDates().remove(curLastModDateType);
    }

    // Set the last modified date with the given DateType.
    tm.addDate(lastModDate);
    hashCode = 0;
  }

  public void setLastModifiedDate(Date lastModDate) {
    if (lastModDate == null)
      throw new IllegalArgumentException("Last modified date can't be null.");

    // Set the last modified date with the given Date.
    DateType lastModDateType = new DateType(false, lastModDate);
    lastModDateType.setType("modified");
    setLastModifiedDate(lastModDateType);
  }

  public void setServiceName(String serviceName) {
    tm.setServiceName(serviceName);
    hashCode = 0;
  }

  // LOOK these are wrong
  public void setContributors(List<ThreddsMetadata.Contributor> a) {
    List<ThreddsMetadata.Contributor> dest = tm.getContributors();
    for (ThreddsMetadata.Contributor item : a) {
      if (!dest.contains(item))
        dest.add(item);
    }
    hashCode = 0;
  }

  public void setKeywords(List<ThreddsMetadata.Vocab> a) {
    List<ThreddsMetadata.Vocab> dest = tm.getKeywords();
    for (ThreddsMetadata.Vocab item : a) {
      if (!dest.contains(item))
        dest.add(item);
    }
    hashCode = 0;
  }

  public void setProjects(List<ThreddsMetadata.Vocab> a) {
    List<ThreddsMetadata.Vocab> dest = tm.getProjects();
    for (ThreddsMetadata.Vocab item : a) {
      if (!dest.contains(item))
        dest.add(item);
    }
    hashCode = 0;
  }

  public void setPublishers(List<ThreddsMetadata.Source> a) {
    List<ThreddsMetadata.Source> dest = tm.getPublishers();
    for (ThreddsMetadata.Source item : a) {
      if (!dest.contains(item))
        dest.add(item);
    }
    hashCode = 0;
  }

  public void setResourceControl(String restrictAccess) {
    this.restrictAccess = restrictAccess;
  }

  //////////////////////////////////////////////////////////////////////////////
  // add/remove/get/find local elements

  /**
   * Add InvAccess element to this dataset.
   *
   * @param a add dthis
   */
  public void addAccess(InvAccess a) {
    accessLocal.add(a);
    hashCode = 0;
  }

  /**
   * Add a list of InvAccess elements to this dataset.
   *
   * @param a add all of these
   */
  public void addAccess(List<InvAccess> a) {
    accessLocal.addAll(a);
    hashCode = 0;
  }

  /**
   * @return the local access (non-expanded) elements.
   */
  public java.util.List<InvAccess> getAccessLocal() {
    return accessLocal;
  }

  /**
   * @return the ncml element if it exists, else return null.
   */
  public org.jdom2.Element getNcmlElement() {
    return ncmlElement;
  }

  public void setNcmlElement(org.jdom2.Element ncmlElement) {
    this.ncmlElement = ncmlElement;
  }

  /**
   * Add a nested dataset.
   *
   * @param ds add this
   */
  public void addDataset(InvDatasetImpl ds) {
    if (ds == null) return;
    ds.setParent(this);
    datasets.add(ds);
    hashCode = 0;
  }

  /**
   * Add a nested dataset at the location indicated by index.
   *
   * @param index add at this position
   * @param ds    add this
   */
  public void addDataset(int index, InvDatasetImpl ds) {
    if (ds == null) return;
    ds.setParent(this);
    datasets.add(index, ds);
    hashCode = 0;
  }

  /**
   * Remove the given dataset element from this dataset if it is in the dataset.
   *
   * @param ds - the dataset element to be removed
   * @return true if this dataset contained the given dataset element.
   */
  public boolean removeDataset(InvDatasetImpl ds) {
    if (this.datasets.remove(ds)) {
      ds.setParent(null);
      InvCatalogImpl cat = (InvCatalogImpl) getParentCatalog();
      if (cat != null)
        cat.removeDatasetByID(ds);
      return (true);
    }
    return (false);
  }

  /**
   * Replace the given dataset if it is a nesetd dataset.
   *
   * @param remove - the dataset element to be removed
   * @param add    - the dataset element to be added
   * @return true on success
   */
  public boolean replaceDataset(InvDatasetImpl remove, InvDatasetImpl add) {
    for (int i = 0; i < datasets.size(); i++) {
      InvDataset dataset = datasets.get(i);
      if (dataset.equals(remove)) {
        datasets.set(i, add);
        InvCatalogImpl cat = (InvCatalogImpl) getParentCatalog();
        if (cat != null) {
          cat.removeDatasetByID(remove);
          cat.addDatasetByID(add);
        }
        return true;
      }
    }
    return false;
  }

  /**
   * Add documentation element to this dataset.
   *
   * @param doc add this
   */
  public void addDocumentation(InvDocumentation doc) {
    tm.addDocumentation(doc);
    hashCode = 0;
  }

  /**
   * Add a property to this dataset
   *
   * @param p add this
   */
  public void addProperty(InvProperty p) {
    tm.addProperty(p);
    hashCode = 0;
  }

  /**
   * Add a service to this dataset.
   *
   * @param service add this service to the dataset
   * @deprecated add services only to catalog
   */
  public void addService(InvService service) {
    // System.out.println("--add dataset service= "+service.getName());
    servicesLocal.add(service);
    services.add(service);
    // add nested servers
    for (InvService nested : service.getServices()) {
      services.add(nested);
      // System.out.println("--add expanded service= "+nested.getName());
    }
    hashCode = 0;
  }

  /**
   * Remove a service from this dataset.
   *
   * @param service remove this
   * @deprecated put services in catalog
   */
  public void removeService(InvService service) {
    servicesLocal.remove(service);
    services.remove(service);
    // remove nested servers
    for (InvService nested : service.getServices()) {
      services.remove(nested);
    }
  }

  /**
   * Get services attached specifically to this dataset.
   *
   * @return List of type InvService. May be empty, but not null.
   */
  public java.util.List<InvService> getServicesLocal() {
    return servicesLocal;
  }

  /**
   * Set the list of services attached specifically to this dataset.
   * Discard any previous servies.
   *
   * @param s list of services.
   */
  public void setServicesLocal(java.util.List<InvService> s) {
    this.services = new ArrayList<>();
    this.servicesLocal = new ArrayList<>();

    for (InvService elem : s) {
      addService(elem);
    }
    hashCode = 0;
  }

  /**
   * Get the metadata stored in this dataset element.
   * Inherited metadata only in an InvMetadata object.
   *
   * @return the metadata stored in this dataset element.
   */
  public ThreddsMetadata getLocalMetadata() {
    return tm;
  }

  public void setLocalMetadata(ThreddsMetadata tm) {
    // look this is wrong.
    // need to copy fields into it !!
    // possible only the one that are different from default !!!
    // like stored defaults !! ha ha !
    this.tm = tm;
    hashCode = 0;
  }

  /**
   * local metadata that should be inherited by this dataset's children.
   *
   * @return local metadata that should be inherited by this dataset's children.
   */
  public ThreddsMetadata getLocalMetadataInheritable() {
    return tmi;
  }

  /*
   * local metadata that should be inherited by this dataset's children.
   *
  public ThreddsMetadata getCat6Metadata() {
    return tmi6;
  } */

  /**
   * Remove the given InvMetadata from the set of metadata local to this dataset.
   *
   * @param metadata remove this
   * @return true if an InvMetadata is removed, false otherwise.
   */
  public boolean removeLocalMetadata(InvMetadata metadata) {
    InvDatasetImpl parentDataset = ((InvDatasetImpl) metadata.getParentDataset());
    List localMdata = parentDataset.getLocalMetadata().getMetadata();
    if (localMdata.contains(metadata)) {
      if (localMdata.remove(metadata)) {
        hashCode = 0; // Need to recalculate the hash code.
        return (true);
      }
    }
    return (false);
  }

  public String getServiceName() {
    if (defaultService != null)
      return defaultService.getName();
    return null;
  }

  /*
   * get Documentation that are xlinks
   *
  public List getDocumentationLinks() {
    ArrayList result = new ArrayList();
    java.util.List docs = getDocumentation();
    for (int i = 0; i < docs.size(); i++) {
      InvDocumentation doc = (InvDocumentation) docs.get(i);
      if (doc.hasXlink())
        result.add(doc);
    }
    return result;
  } */

  /**
   * Filtering
   *
   * @return true if this is "marked"
   */
  protected boolean getMark() {
    return mark;
  }

  protected void setMark(boolean mark) {
    this.mark = mark;
  }

  /**
   * Look up the User property having the given key
   *
   * @param key property key
   * @return User property having the given key, or null
   */
  public Object getUserProperty(Object key) {
    if (userMap == null) return null;
    return userMap.get(key);
  }

  public void setUserProperty(Object key, Object value) {
    if (userMap == null) userMap = new HashMap<>();
    userMap.put(key, value);
  }

  private HashMap<Object, Object> userMap = null;

  public String toString() {
    return getName();
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * @param buff          a
   * @param ds            a
   * @param complete      a
   * @param isServer      a
   * @param datasetEvents a
   * @param catrefEvents  a
   * @deprecated Instead use {@link #writeHtmlDescription(StringBuilder buff, InvDatasetImpl ds, boolean complete, boolean isServer, boolean datasetEvents, boolean catrefEvents, boolean resolveRelativeUrls)}
   */
  static public void writeHtmlDescription(StringBuilder buff, InvDatasetImpl ds,
                                          boolean complete, boolean isServer,
                                          boolean datasetEvents,
                                          boolean catrefEvents) {
    writeHtmlDescription(buff, ds, complete, isServer, datasetEvents, catrefEvents, true);
  }

  /**
   * Write an Html representation of the given dataset.
   * <p> With datasetEvents, catrefEvents = true, this is used to construct an HTML page on the client
   * (eg using HtmlPage); the client then detects URL clicks and processes.
   * <p> With datasetEvents, catrefEvents = false, this is used to construct an HTML page on the server.
   * (eg using HtmlPage); the client then detects URL clicks and processes.
   *
   * @param buff          put HTML here.
   * @param ds            the dataset.
   * @param complete      if true, add HTML header and ender so its a complete, valid HTML page.
   * @param isServer      if true, then we are in the thredds data server, so do the following: <ul>
   *                      <li> append "html" to DODS Access URLs
   *                      </ul>
   * @param datasetEvents if true, prepend "dataset:" to any dataset access URLS
   * @param catrefEvents  if true, prepend "catref:" to any catref URLS
   */

  static public void writeHtmlDescription(StringBuilder buff, InvDatasetImpl ds,
                                          boolean complete, boolean isServer,
                                          boolean datasetEvents, boolean catrefEvents,
                                          boolean resolveRelativeUrls) {

    if (ds == null) return;

    if (complete) {
      buff.append("<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\"\n")
              .append("        \"http://www.w3.org/TR/html4/loose.dtd\">\n")
              .append("<html>\n");
      buff.append("<head>");
      buff.append("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">");
      buff.append("</head>");
      buff.append("<body>\n");
    }

    buff.append("<h2>Dataset: ").append(ds.getFullName()).append("</h2>\n<ul>\n");
    if ((ds.getDataFormatType() != null) && (ds.getDataFormatType() != DataFormatType.NONE))
      buff.append(" <li><em>Data format: </em>").append(StringUtil2.quoteHtmlContent(ds.getDataFormatType().toString())).append("</li>\n");

    if ((ds.getDataSize() != 0.0) && !Double.isNaN(ds.getDataSize()))
      buff.append(" <li><em>Data size: </em>").append(Format.formatByteSize(ds.getDataSize())).append("</li>\n");

    if ((ds.getDataType() != null) && (ds.getDataType() != FeatureType.ANY))
      buff.append(" <li><em>Data type: </em>").append(StringUtil2.quoteHtmlContent(ds.getDataType().toString())).append("</li>\n");

    if ((ds.getCollectionType() != null) && (ds.getCollectionType() != CollectionType.NONE))
      buff.append(" <li><em>Collection type: </em>").append(StringUtil2.quoteHtmlContent(ds.getCollectionType().toString())).append("</li>\n");

    if (ds.isHarvest())
      buff.append(" <li><em>Harvest: </em>").append(ds.isHarvest()).append("</li>\n");

    if (ds.getAuthority() != null)
      buff.append(" <li><em>Naming Authority: </em>").append(StringUtil2.quoteHtmlContent(ds.getAuthority())).append("</li>\n");

    if (ds.getID() != null)
      buff.append(" <li><em>ID: </em>").append(StringUtil2.quoteHtmlContent(ds.getID())).append("</li>\n");

    if (ds.getRestrictAccess() != null)
      buff.append(" <li><em>RestrictAccess: </em>").append(StringUtil2.quoteHtmlContent(ds.getRestrictAccess())).append("</li>\n");

    if (ds instanceof InvCatalogRef) {
      InvCatalogRef catref = (InvCatalogRef) ds;
      String href = resolveRelativeUrls || catrefEvents
              ? resolve(ds, catref.getXlinkHref())
              : catref.getXlinkHref();
      if (catrefEvents) href = "catref:" + href;
      buff.append(" <li><em>CatalogRef: </em>").append(makeHref(href, null)).append("</li>\n");
    }

    buff.append("</ul>\n");

    java.util.List<InvDocumentation> docs = ds.getDocumentation();
    if (docs.size() > 0) {
      buff.append("<h3>Documentation:</h3>\n<ul>\n");
      for (InvDocumentation doc : docs) {
        String type = (doc.getType() == null) ? "" : "<strong>" + StringUtil2.quoteHtmlContent(doc.getType()) + ":</strong> ";
        String inline = doc.getInlineContent();
        if ((inline != null) && (inline.length() > 0))
          buff.append(" <li>").append(type).append(StringUtil2.quoteHtmlContent(inline)).append("</li>\n");
        if (doc.hasXlink()) {
          // buff.append(" <li>" + type + makeHrefResolve(ds, url.toString(), doc.getXlinkTitle()) + "</a>\n");
          buff.append(" <li>").append(type).append(makeHref(doc.getXlinkHref(), doc.getXlinkTitle())).append("</li>\n");
        }
      }
      buff.append("</ul>\n");
    }

    java.util.List<InvAccess> access = ds.getAccess();
    if (access.size() > 0) {
      buff.append("<h3>Access:</h3>\n<ol>\n");
      for (InvAccess a : access) {
        InvService s = a.getService();
        String urlString = resolveRelativeUrls || datasetEvents
                ? a.getStandardUrlName()
                : a.getUnresolvedUrlName();
        String fullUrlString = urlString;
        if (datasetEvents) fullUrlString = "dataset:" + fullUrlString;
        if (isServer) {
          ServiceType stype = s.getServiceType();
          if ((stype == ServiceType.OPENDAP) || (stype == ServiceType.DODS))
            fullUrlString = fullUrlString + ".html";
          else if (stype == ServiceType.DAP4)
            fullUrlString = fullUrlString + ".dmr.xml";
          else if (stype == ServiceType.WCS)
            fullUrlString = fullUrlString + "?service=WCS&version=1.0.0&request=GetCapabilities";
          else if (stype == ServiceType.WMS)
            fullUrlString = fullUrlString + "?service=WMS&version=1.3.0&request=GetCapabilities";
          //NGDC update 8/18/2011
          else if (stype == ServiceType.NCML || stype == ServiceType.UDDC || stype == ServiceType.ISO) {
            String catalogUrl = ds.getCatalogUrl();
        	  String datasetId = ds.id;
            if ( catalogUrl.indexOf('#') > 0)
              catalogUrl = catalogUrl.substring( 0, catalogUrl.lastIndexOf('#'));
            try {
              catalogUrl = URLEncoder.encode( catalogUrl, "UTF-8" );
              datasetId = URLEncoder.encode( datasetId,"UTF-8");
            } catch ( UnsupportedEncodingException e) {
              e.printStackTrace();
            }
            fullUrlString = fullUrlString + "?catalog=" + catalogUrl +  "&dataset=" + datasetId;
          }
          else if (stype == ServiceType.NetcdfSubset)
            fullUrlString = fullUrlString + "/dataset.html";
          else if ((stype == ServiceType.CdmRemote) || (stype == ServiceType.CdmrFeature))
            fullUrlString = fullUrlString + "?req=form";
        }
        buff.append(" <li> <b>").append(StringUtil2.quoteHtmlContent(s.getServiceType().toString()));
        buff.append(":</b> ").append(makeHref(fullUrlString, urlString)).append("</li>\n");
      }
      buff.append("</ol>\n");
    }

    java.util.List<ThreddsMetadata.Contributor> contributors = ds.getContributors();
    if (contributors.size() > 0) {
      buff.append("<h3>Contributors:</h3>\n<ul>\n");
      for (ThreddsMetadata.Contributor t : contributors) {
        String role = (t.getRole() == null) ? "" : "<strong> (" + StringUtil2.quoteHtmlContent(t.getRole()) + ")</strong> ";
        buff.append(" <li>").append(StringUtil2.quoteHtmlContent(t.getName())).append(role).append("</li>\n");
      }
      buff.append("</ul>\n");
    }

    java.util.List<ThreddsMetadata.Vocab> keywords = ds.getKeywords();
    if (keywords.size() > 0) {
      buff.append("<h3>Keywords:</h3>\n<ul>\n");
      for (ThreddsMetadata.Vocab t : keywords) {
        String vocab = (t.getVocabulary() == null) ? "" : " <strong>(" + StringUtil2.quoteHtmlContent(t.getVocabulary()) + ")</strong> ";
        buff.append(" <li>").append(StringUtil2.quoteHtmlContent(t.getText())).append(vocab).append("</li>\n");
      }
      buff.append("</ul>\n");
    }

    java.util.List<DateType> dates = ds.getDates();
    if (dates.size() > 0) {
      buff.append("<h3>Dates:</h3>\n<ul>\n");
      for (DateType d : dates) {
        String type = (d.getType() == null) ? "" : " <strong>(" + StringUtil2.quoteHtmlContent(d.getType()) + ")</strong> ";
        buff.append(" <li>").append(StringUtil2.quoteHtmlContent(d.getText())).append(type).append("</li>\n");
      }
      buff.append("</ul>\n");
    }

    java.util.List<ThreddsMetadata.Vocab> projects = ds.getProjects();
    if (projects.size() > 0) {
      buff.append("<h3>Projects:</h3>\n<ul>\n");
      for (ThreddsMetadata.Vocab t : projects) {
        String vocab = (t.getVocabulary() == null) ? "" : " <strong>(" + StringUtil2.quoteHtmlContent(t.getVocabulary()) + ")</strong> ";
        buff.append(" <li>").append(StringUtil2.quoteHtmlContent(t.getText())).append(vocab).append("</li>\n");
      }
      buff.append("</ul>\n");
    }

    java.util.List<ThreddsMetadata.Source> creators = ds.getCreators();
    if (creators.size() > 0) {
      buff.append("<h3>Creators:</h3>\n<ul>\n");
      for (ThreddsMetadata.Source t : creators) {
        buff.append(" <li><strong>").append(StringUtil2.quoteHtmlContent(t.getName())).append("</strong><ul>\n");
        buff.append(" <li><em>email: </em>").append(StringUtil2.quoteHtmlContent(t.getEmail())).append("</li>\n");
        if (t.getUrl() != null) {
          String newUrl = resolveRelativeUrls
                  ? makeHrefResolve(ds, t.getUrl(), null)
                  : makeHref(t.getUrl(), null);
          buff.append(" <li> <em>").append(newUrl).append("</em></li>\n");
        }
        buff.append(" </ul></li>\n");
      }
      buff.append("</ul>\n");
    }

    java.util.List<ThreddsMetadata.Source> publishers = ds.getPublishers();
    if (publishers.size() > 0) {
      buff.append("<h3>Publishers:</h3>\n<ul>\n");
      for (ThreddsMetadata.Source t : publishers) {
        buff.append(" <li><strong>").append(StringUtil2.quoteHtmlContent(t.getName())).append("</strong><ul>\n");
        buff.append(" <li><em>email: </em>").append(StringUtil2.quoteHtmlContent(t.getEmail())).append("\n");
        if (t.getUrl() != null) {
          String urlLink = resolveRelativeUrls
                  ? makeHrefResolve(ds, t.getUrl(), null)
                  : makeHref(t.getUrl(), null);
          buff.append(" <li> <em>").append(urlLink).append("</em>\n");
        }
        buff.append(" </ul>\n");
      }
      buff.append("</ul>\n");
    }

    /*
    4.2:
    <h3>Variables:</h3>
    <ul>
    <li><em>Vocabulary</em> [DIF]:
    <ul>
     <li><strong>Reflectivity</strong> =  <i></i> = EARTH SCIENCE > Spectral/Engineering > Radar > Radar Reflectivity (db)
     <li><strong>Velocity</strong> =  <i></i> = EARTH SCIENCE > Spectral/Engineering > Radar > Doppler Velocity (m/s)
     <li><strong>SpectrumWidth</strong> =  <i></i> = EARTH SCIENCE > Spectral/Engineering > Radar > Doppler Spectrum Width (m/s)
     </ul>
     </ul>
    </ul>

    4.3:
    <h3>Variables:</h3>
    <ul>
    <li><em>Vocabulary</em> [CF-1.0]:
    <ul>
     <li><strong>d3d (meters) </strong> =  <i>3D Depth at Nodes
    <p>        </i> = depth_at_nodes
     <li><strong>depth (meters) </strong> =  <i>Bathymetry</i> = depth
     <li><strong>eta (m) </strong> =  <i></i> =
     <li><strong>temp (Celsius) </strong> =  <i>Temperature
    <p>        </i> = sea_water_temperature
     <li><strong>u (m/s) </strong> =  <i>Eastward Water
    <p>          Velocity
    <p>        </i> = eastward_sea_water_velocity
     <li><strong>v (m/s) </strong> =  <i>Northward Water
    <p>          Velocity
    <p>        </i> = northward_sea_water_velocity
    </ul>
    </ul>
     */

    java.util.List<ThreddsMetadata.Variables> vars = ds.getVariables();
    if (vars.size() > 0) {
      buff.append("<h3>Variables:</h3>\n<ul>\n");
      for (ThreddsMetadata.Variables t : vars) {

        buff.append("<li><em>Vocabulary</em> [");
        if (t.getVocabUri() != null) {
          URI uri = t.getVocabUri();
          String vocabLink = resolveRelativeUrls
                  ? makeHrefResolve(ds, uri.toString(), t.getVocabulary())
                  : makeHref(uri.toString(), t.getVocabulary());
          buff.append(vocabLink);
        } else {
          buff.append(StringUtil2.quoteHtmlContent(t.getVocabulary()));
        }
        buff.append("]:\n<ul>\n");

        java.util.List<ThreddsMetadata.Variable> vlist = t.getVariableList();
        if (vlist.size() > 0) {
          for (ThreddsMetadata.Variable v : vlist) {
            String units = (v.getUnits() == null || v.getUnits().length() == 0) ? "" : " (" + v.getUnits() + ") ";
            buff.append(" <li><strong>").append(StringUtil2.quoteHtmlContent(v.getName() + units)).append("</strong> = ");
            String desc = (v.getDescription() == null) ? "" : " <i>" + StringUtil2.quoteHtmlContent(v.getDescription()) + "</i> = ";
            buff.append(desc);
            if (v.getVocabularyName() != null)
              buff.append(StringUtil2.quoteHtmlContent(v.getVocabularyName()));
            buff.append("\n");
          }
        }
        buff.append("</ul>\n");
      }
      buff.append("</ul>\n");
    }
    if (ds.getVariableMapLink() != null) {
      buff.append("<h3>Variables:</h3>\n");
      buff.append("<ul><li>"+makeHref(ds.getVariableMapLink(), "VariableMap")+"</li></ul>\n");
    }

    ThreddsMetadata.GeospatialCoverage gc = ds.getGeospatialCoverage();
    if ((gc != null) && !gc.isEmpty()) {
      buff.append("<h3>GeospatialCoverage:</h3>\n<ul>\n");
      if (gc.isGlobal())
        buff.append(" <li><em> Global </em>\n");

      buff.append(" <li><em> Longitude: </em> ").append(rangeString(gc.getEastWestRange())).append("</li>\n");
      buff.append(" <li><em> Latitude: </em> ").append(rangeString(gc.getNorthSouthRange())).append("</li>\n");
      if (gc.getUpDownRange() != null) {
        buff.append(" <li><em> Altitude: </em> ").append(rangeString(gc.getUpDownRange())).append(" (positive is <strong>").append(StringUtil2.quoteHtmlContent(gc.getZPositive())).append(")</strong></li>\n");
      }

      java.util.List<ThreddsMetadata.Vocab> nlist = gc.getNames();
      if ((nlist != null) && (nlist.size() > 0)) {
        buff.append(" <li><em>  Names: </em> <ul>\n");
        for (ThreddsMetadata.Vocab elem : nlist) {
          buff.append(" <li>").append(StringUtil2.quoteHtmlContent(elem.getText())).append("\n");
        }
        buff.append(" </ul>\n");
      }
      buff.append(" </ul>\n");
    }

    CalendarDateRange tc = ds.getCalendarDateCoverage();
    if (tc != null) {
      buff.append("<h3>TimeCoverage:</h3>\n<ul>\n");
      CalendarDate start = tc.getStart();
      if (start != null)
        buff.append(" <li><em>  Start: </em> ").append(start.toString()).append("\n");
      CalendarDate end = tc.getEnd();
      if (end != null) {
        buff.append(" <li><em>  End: </em> ").append(end.toString()).append("\n");
      }
      CalendarDuration duration = tc.getDuration();
      if (duration != null)
        buff.append(" <li><em>  Duration: </em> ").append(StringUtil2.quoteHtmlContent(duration.toString())).append("\n");
      CalendarDuration resolution = tc.getResolution();
      if (resolution != null) {
        buff.append(" <li><em>  Resolution: </em> ").append(StringUtil2.quoteHtmlContent(resolution.toString())).append("\n");
      }
      buff.append(" </ul>\n");
    }

    java.util.List<InvMetadata> metadata = ds.getMetadata();
    boolean gotSomeMetadata = false;
    for (InvMetadata m : metadata) {
      if (m.hasXlink()) gotSomeMetadata = true;
    }

    if (gotSomeMetadata) {
      buff.append("<h3>Metadata:</h3>\n<ul>\n");
      for (InvMetadata m : metadata) {
        String type = (m.getMetadataType() == null) ? "" : m.getMetadataType();
        if (m.hasXlink()) {
          String title = (m.getXlinkTitle() == null) ? "Type " + type : m.getXlinkTitle();
          String mdLink = resolveRelativeUrls
                  ? makeHrefResolve(ds, m.getXlinkHref(), title)
                  : makeHref(m.getXlinkHref(), title);
          buff.append(" <li> ").append(mdLink).append("\n");
        } //else {
        //buff.append(" <li> <pre>"+m.getMetadataType()+" "+m.getContentObject()+"</pre>\n");
        //}
      }
      buff.append("</ul>\n");
    }

    java.util.List<InvProperty> propsOrg = ds.getProperties();
    java.util.List<InvProperty> props = new ArrayList<>(ds.getProperties().size());
    for (InvProperty p : propsOrg)  {
      if (!p.getName().startsWith("viewer"))  // eliminate the viewer properties from the html view
        props.add(p);
    }
    if (props.size() > 0) {
      buff.append("<h3>Properties:</h3>\n<ul>\n");
      for (InvProperty p : props) {
        if (p.getName().equals("attachments")) // LOOK whats this ?
        {
          String attachLink = resolveRelativeUrls
                  ? makeHrefResolve(ds, p.getValue(), p.getName())
                  : makeHref(p.getValue(), p.getName());
          buff.append(" <li>").append(attachLink).append("\n");
        } else {
          buff.append(" <li>").append(StringUtil2.quoteHtmlContent(p.getName() + " = \"" + p.getValue())).append("\"\n");
        }
      }
      buff.append("</ul>\n");
    }

    if (complete) buff.append("</body></html>");
  }

  static private String rangeString(ThreddsMetadata.Range r) {
    if (r == null) return "";
    String units = (r.getUnits() == null) ? "" : " " + r.getUnits();
    String resolution = r.hasResolution() ? " Resolution=" + r.getResolution() : "";
    return StringUtil2.quoteHtmlContent(r.getStart() + " to " + (r.getStart() + r.getSize()) + resolution + units);
  }

  /**
   * resolve reletive URLS against the catalog URL.
   *
   * @param ds   use ds parent catalog, if it exists
   * @param href URL to resolve
   * @return resolved URL
   */
  static public String resolve(InvDataset ds, String href) {
    InvCatalog cat = ds.getParentCatalog();
    if (cat != null) {
      try {
        java.net.URI uri = cat.resolveUri(href);
        href = uri.toString();
      } catch (java.net.URISyntaxException e) {
        logger.warn("InvDatasetImpl.writeHtml: error parsing URL= " + href);
      }
    }
    return href;
  }

  static private String makeHref(String href, String title) {
    if (title == null) title = href;
    return "<a href='" + StringUtil2.quoteHtmlContent(href) + "'>" + StringUtil2.quoteHtmlContent(title) + "</a>";
  }

  static private String makeHrefResolve(InvDatasetImpl ds, String href, String title) {
    if (title == null) title = href;
    href = resolve(ds, href);
    return makeHref(href, title);
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * @return debugging info
   */
  public String dump() {
    return dump(0);
  }

  String dump(int n) {
    StringBuilder buff = new StringBuilder(100);

    buff.append(indent(n));
    buff.append("Dataset name:<").append(getName());
    if (dataType != null) {
      buff.append("> dataType:<").append(dataType);
    }
    if (urlPath != null)
      buff.append("> urlPath:<").append(urlPath);
    if (defaultService != null)
      buff.append("> defaultService <").append(defaultService);
    buff.append("> uID:<").append(getUniqueID());
    buff.append(">\n");

    List<InvService> svcs = getServicesLocal();
    if (svcs.size() > 0) {
      String indent = indent(n + 2);
      buff.append(indent);
      buff.append("Services:\n");
      for (InvService s : svcs) {
        buff.append(s.dump(n + 4));
      }
    }

    if (access.size() > 0) {
      String indent = indent(n + 2);
      buff.append(indent);
      if (access.size() == 1) {
        buff.append("Access: ").append(access.get(0)).append("\n");
      } else if (access.size() > 1) {
        buff.append("Access:\n");
        for (InvAccess a : access) {
          buff.append(indent(n + 4)).append(a).append("\n");
        }
      }
    }

    buff.append(indent(n)).append("Thredds Metadata\n");
    buff.append(tm.dump(n + 4)).append("\n");
    buff.append(indent(n)).append("Thredds Metadata Inherited\n");
    buff.append(tmi.dump(n + 4)).append("\n");
    //buff.append(indent(n)).append("Thredds Metadata Cat6\n");
    //buff.append(tmi6.dump(n + 4)).append("\n");

    if (datasets.size() > 0) {
      String indent = indent(n + 2);
      buff.append(indent);
      buff.append("Datasets:\n");
      for (InvDataset ds : datasets) {
        InvDatasetImpl dsi = (InvDatasetImpl) ds;
        buff.append(dsi.dump(n + 4));
      }
    }
    return buff.toString();
  }

  static String indent(int n) {
    StringBuilder blanks = new StringBuilder(n);
    for (int i = 0; i < n; i++)
      blanks.append(" ");
    return blanks.toString();
  }

  boolean check(StringBuilder out, boolean show) {
    boolean isValid = true;

    if (log.length() > 0) {
      out.append(log);
    }

    /* check that the serviceName is valid
    if (serviceName != null) {
      if (null == findService(serviceName)) {
        out.append("**Dataset ("+getFullName()+"): has unknown service named ("+serviceName+")\n");
        isValid = false;
      }
    } */

    for (InvAccess acces : access) {
      InvAccessImpl a = (InvAccessImpl) acces;
      isValid &= a.check(out, show);
    }

    for (InvDataset dataset : datasets) {
      InvDatasetImpl ds = (InvDatasetImpl) dataset;
      isValid &= ds.check(out, show);
    }

    for (InvMetadata m : getMetadata()) {
      m.check(out);
    }

    for (InvService s : getServicesLocal()) {
      isValid &= s.check(out);
    }

    if (hasAccess() && (getDataType() == null)) {
      out.append("**Warning: Dataset (").append(getFullName()).append("): is selectable but no data type declared in it or in a parent element\n");
    }

    if (!hasAccess() && !hasNestedDatasets()) {
      out.append("**Warning: Dataset (").append(getFullName()).append("): is not selectable and does not have nested datasets\n");
    }

    if (show) System.out.println("  dataset " + name + " valid = " + isValid);

    return isValid;
  }

  /**
   * InvDatasetImpl elements with same values are equal.
   */
  public boolean equals(Object o) {
    if (this == o) return true;
    if (!(o instanceof InvDatasetImpl)) return false;
    return o.hashCode() == this.hashCode();
  }

  /**
   * Override Object.hashCode() to implement equals.
   */
  public int hashCode() {
    if (hashCode == 0) {
      int result = 17;
      result = 37 * result + getName().hashCode();
      result = 37 * result + getServicesLocal().hashCode();
      result = 37 * result + getDatasets().hashCode();
      result = 37 * result + getAccessLocal().hashCode();
      result = 37 * result + (isHarvest() ? 1 : 0);

      if (null != getCollectionType())
        result = 37 * result + getCollectionType().hashCode();

      result = 37 * result + getDocumentation().hashCode();
      result = 37 * result + getProperties().hashCode();
      result = 37 * result + getMetadata().hashCode();

      result = 37 * result + getCreators().hashCode();
      result = 37 * result + getContributors().hashCode();
      result = 37 * result + getDates().hashCode();
      result = 37 * result + getKeywords().hashCode();
      result = 37 * result + getProjects().hashCode();
      result = 37 * result + getPublishers().hashCode();
      result = 37 * result + getVariables().hashCode();

      if (null != getID())
        result = 37 * result + getID().hashCode();
      if (null != getAlias())
        result = 37 * result + getAlias().hashCode();
      if (null != getAuthority())
        result = 37 * result + getAuthority().hashCode();
      if (null != getDataType())
        result = 37 * result + getDataType().hashCode();
      if (null != getDataFormatType())
        result = 37 * result + getDataFormatType().hashCode();
      if (null != getServiceDefault())
        result = 37 * result + getServiceDefault().hashCode();
      if (null != getUrlPath())
        result = 37 * result + getUrlPath().hashCode();

      if (null != getGeospatialCoverage())
        result = 37 * result + getGeospatialCoverage().hashCode();

      if (null != getCalendarDateCoverage())
        result = 37 * result + getCalendarDateCoverage().hashCode(); // */

      hashCode = result;
    }
    return hashCode;
  }

  private volatile int hashCode = 0; // Bloch, item 8 - lazily initialize hash value

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.cataloggen.config;

import thredds.catalog.InvDatasetImpl;
import thredds.catalog.InvDataset;

import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 *
Title: Catalog Generator

 *
Description: Tool for generating THREDDS catalogs.

 *
Copyright: Copyright (c) 2001

 *
Company: UCAR/Unidata

 *
 * @author Ethan Davis
 * @version 1.0
 */

public class DatasetNamer {
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetNamer.class);

  private InvDataset parentDataset = null;

  // attributes of the datasetNamer element
  private String name = null;
  private boolean addLevel = false;
  private DatasetNamerType type = null;
  private String matchPattern = null;
  protected Pattern regExpPattern;

  private String substitutePattern = null;
  private String attribContainer = null;
  private String attribName = null;

  // for validation
  private boolean isValid = true;
  private StringBuffer msgLog = new StringBuffer();

  /**
   * Constructor
   *
   * @param parentDs
   * @param name
   * @param addLevelBoolean
   * @param typeName
   * @param matchPattern
   * @param substitutePattern
   * @param attribContainer
   * @param attribName
   */
  public DatasetNamer( InvDataset parentDs, String name, String addLevelBoolean, String typeName,
                       String matchPattern, String substitutePattern, String attribContainer, String attribName)
  {
    this( parentDs, name, Boolean.parseBoolean(addLevelBoolean), DatasetNamerType.getType(typeName),
          matchPattern, substitutePattern, attribContainer, attribName);
    // Check that type is not null.
    if ( this.getType() == null ) {
      this.isValid = false;
      msgLog.append( " ** DatasetNamer (1): invalid type [" + typeName + "] for datasetNamer [" + name + "]." );
    }
  }

  /**
   * Constructor
   *
   * @param parentDs
   * @param name
   * @param addLevel
   * @param type
   * @param matchPattern
   * @param substitutePattern
   * @param attribContainer
   * @param attribName
   */
  public DatasetNamer( InvDataset parentDs, String name, boolean addLevel, DatasetNamerType type,
                       String matchPattern, String substitutePattern, String attribContainer, String attribName)
  {
    this.parentDataset = parentDs;
    this.name = name;
    this.addLevel = addLevel;
    if ( type == null ) {
      this.isValid = false;
      msgLog.append( " ** DatasetNamer (1): null type for datasetNamer [" + name + "]." );
    }
    this.type = type;

    if ( matchPattern != null )
    {
      this.matchPattern = matchPattern;
      if ( DatasetNamerType.REGULAR_EXPRESSION.equals( this.type ))
      {
        try
        {
          this.regExpPattern = java.util.regex.Pattern.compile( this.matchPattern );
        }
        catch ( PatternSyntaxException e )
        {
          isValid = false;
          msgLog.append( " ** DatasetNamer (3): invalid matchPattern [" + this.matchPattern + "]." );
        }
      }
    }

    this.substitutePattern = substitutePattern;
    this.attribContainer = attribContainer;
    this.attribName = attribName;
  }

  /**
   * Return the parent dataset of this DatasetNamer
   */
  public InvDataset getParentDataset() {
    return (this.parentDataset);
  }

  /**
   * Return the name of this DatasetNamer.
   *
   * @return String name of this DatasetNamer
   */
  public String getName() {
    return (this.name);
  }

  /**
   * Return the value of the addLevel attribute of this DatasetNamer.
   *
   * @return boolean addLevel attribute value for this DatasetNamer
   */
  public boolean getAddLevel() {
    return (this.addLevel);
  }

  /**
   * Return the type attribute of this DatasetNamer.
   *
   * @return DatasetNamerType type value for this DatasetNamer
   */
  public DatasetNamerType getType() {
    return (this.type);
  }

  /**
   * Return the value of the matchPattern attribute of this DatasetNamer.
   *
   * @return String value of matchPattern
   */
  public String getMatchPattern() {
    return (this.matchPattern);
  }

  /**
   * Return the value of the substitutePattern attribute of this DatasetNamer.
   *
   * @return String - value of substitutePattern
   */
  public String getSubstitutePattern() {
    return (this.substitutePattern);
  }

  /**
   * Return the value of the attribContainer attribute of this DatasetNamer.
   *
   * @return String - value of attribContainer
   */
  public String getAttribContainer() {
    return (this.attribContainer);
  }

  /**
   * Return the value of the attribName attribute of this DatasetNamer.
   *
   * @return String - value of attribName
   */
  public String getAttribName() {
    return (this.attribName);
  }

  /**
   * Validate this DatasetNamer object. Return true if valid, false if invalid.
   *
   * @param out StringBuffer with validation messages.
   * @return boolean true if valid, false if invalid
   */
  boolean validate(StringBuilder out) {
    this.isValid = true;

    // If log from construction has content, append to validation output msg.
    if (this.msgLog.length() > 0) {
      out.append(this.msgLog);
    }

    // Check that name is not null (it can be an empty string).
    if (this.getName() == null) {
      this.isValid = false;
      out.append(" ** DatasetNamer (1): null value for name is not valid.");
    }

    // Check that addLevel is not null.
    // boolean can't be null
    //if ( this.getAddLevel() == null)
    //{
    //  this.isValid = false;
    //  out.append(" ** DatasetNamer (2): null value for addLevel is not valid.");
    //}

    // Check that type is not null.
    if (this.getType() == null) {
      this.isValid = false;
      out.append(" ** DatasetNamer (3): null value for type is not valid (set with bad string?).");
    }

    if ( this.getType() == DatasetNamerType.REGULAR_EXPRESSION
         && ( this.getMatchPattern() == null || this.getSubstitutePattern() == null ))
    {
      this.isValid = false;
      out.append(" ** DatasetNamer (4): invalid datasetNamer <" + this.getName() + ">;" +
              " type is " + this.getType().toString() + ": matchPattern(" + this.getMatchPattern() + ") and substitutionPattern(" + this.getSubstitutePattern() + ") " +
              "must not be null.");
    }

    if ( this.getType() == DatasetNamerType.DODS_ATTRIBUTE
            && ( this.getAttribContainer() == null || this.getAttribName() == null ) )
    {
      this.isValid = false;
      out.append(" ** DatasetNamer (5): invalid datasetNamer <" + this.getName() + ">;" +
              " type is " + this.getType().toString() + ": attriuteContainer(" + this.getAttribContainer() + ") and attributeName(" + this.getAttribName() + ") must not be null.");
    }

    return (this.isValid);
  }

  /**
   * string representation
   */
  public String toString() {
    StringBuffer tmp = new StringBuffer();
    tmp.append("DatasetNamer[name:<" + this.getName() + "> addLevel:<" +
            this.getAddLevel() + "> type:<" + this.getType() +
            "> matchPattern:<" + this.getMatchPattern() +
            "> substitutePatter:<" + this.getSubstitutePattern() +
            "> attribContainer:<" + this.getAttribContainer() +
            "> attribName:<" + this.getAttribName() + ">]");
    return (tmp.toString());
  }

  /**
   * Try to name the given dataset.
   */
  public boolean nameDataset(InvDatasetImpl dataset) {
    if (this.type == DatasetNamerType.REGULAR_EXPRESSION) {
      return (this.nameDatasetRegExp(dataset));
    } /* else if (this.type == DatasetNamerType.DODS_ATTRIBUTE) {
      return (this.nameDatasetDodsAttrib(dataset));
    } */
          else {
      String tmpMsg = "This DatasetNamer <" + this.getName() + "> has unsupported type <" + this.type.toString() + ">.";
      logger.error("nameDataset(): " + tmpMsg);
      throw new IllegalStateException(tmpMsg);
    }

  }

  /**
   * Try to name the given dataset.
   */
  public boolean nameDatasetList(java.util.List datasetList)
          throws java.lang.Exception {
    boolean returnValue = false;

    InvDatasetImpl curDataset = null;
    for (int i = 0; i < datasetList.size(); i++) {
      curDataset = (InvDatasetImpl) datasetList.get(i);
      returnValue &= this.nameDataset(curDataset);
    }
    return (returnValue);
  }

  /**  */
  private boolean nameDatasetRegExp(InvDatasetImpl dataset)
  {
    boolean isMatch;
    Matcher matcher;

    // Test for a match on the urlPath
    if (dataset.getUrlPath() != null) {
      logger.debug("nameDatasetRegExp(): try naming on urlPath <{}>", dataset.getUrlPath());
      matcher = this.regExpPattern.matcher( dataset.getUrlPath() );
      isMatch = matcher.find();
    } else {
      matcher = this.regExpPattern.matcher( dataset.getName() );
      isMatch = matcher.find();
    }

    if ( isMatch) {
      // Test for substitution.
      StringBuffer resultingName = new StringBuffer();
      matcher.appendReplacement( resultingName, this.substitutePattern );
      resultingName.delete( 0, matcher.start() );

      if ( resultingName.length() != 0) {
        logger.debug( "nameDatasetRegExp(): Setting name to \"" + resultingName + "\".");
        dataset.setName( resultingName.toString());
        return true;
      } else {
        logger.debug( "nameDatasetRegExp(): No name for regEx substitution.");
        return false;
      }
    }
    if (logger.isDebugEnabled())
      logger.debug("nameDatasetRegExp(): Neither URL <" + dataset.getUrlPath() + "> or name <" +
              dataset.getName() + "> matched pattern <" + this.matchPattern + "> .");
    return false;
  }

  /**
  private boolean nameDatasetDodsAttrib(InvDatasetImpl dataset)
//    throws java.lang.Exception
  {
    DConnect dodsConnection = null;
    DAS das = null;

    boolean acceptDeflate = true;

    String newDatasetName = null;

    //-----
    // Test that this dataset has a DODS type service.
    //-----
    InvAccess access = dataset.getAccess(ServiceType.DODS);

    if (access == null) {
      logger.warn("nameDatasetDodsAttrib(): dataset is not DODS accessible and so cannot be named using DODS attributes.");
      return (false);
    }

    //-----
    // Connect to DODS dataset.
    //-----
    String url = access.getStandardUrlName();

    try {
      dodsConnection = new DConnect(url, acceptDeflate);
    }
    catch (java.io.FileNotFoundException e) {
      logger.error("nameDatasetDodsAttrib(): URL <" + url + "> not found: " + e.getMessage());
      return (false);
    }
    catch (Exception e) // java.lang.NullPointerException
    {
      logger.error("nameDatasetDodsAttrib(): Failed DODS connect: " + e.getMessage());
      return (false);
    }

    logger.debug("nameDatasetDodsAttrib(): Got DODS Connect <url={}>", url);

    //-----
    // Get the DAS
    //-----
    try {
      das = dodsConnection.getDAS();
    }
    catch (DAP2Exception e) // DODSException and DASException
    {
      logger.error("nameDatasetDodsAttrib(): Failed to get DAS: " + e.getMessage());
      return (false);
    }
    catch (opendap.dap.parsers.ParseException e) {
      logger.error("nameDatasetDodsAttrib(): Failed to get DAS: " + e.getMessage());
      return (false);
    }
    catch (Exception e) // java.net.MalformedURLException, java.io.IOException
    {
      logger.error("nameDatasetDodsAttrib(): Failed to get DAS: " + e.getMessage());
      return (false);
    }

    logger.debug("nameDatasetDodsAttrib(): Got DAS");

    //-----
    // Get attribute value from attribute container.
    //-----
    AttributeTable dodsAttTable = null;
    try {
      dodsAttTable = das.getAttributeTable(this.attribContainer);

      if (dodsAttTable != null) {
        Attribute desiredAtt =
                dodsAttTable.getAttribute(this.attribName);
        // Check that desired attribute is a string
        if (desiredAtt.getType() == opendap.dap.Attribute.STRING) {
          java.util.Enumeration enumValues = desiredAtt.getValues();
          if (enumValues.hasMoreElements()) {
            newDatasetName = (String) enumValues.nextElement();
            // Java DODS string attributes are enclosed in double quotes.
            // Remove them.
            newDatasetName = newDatasetName.substring(1, newDatasetName.length() - 1);
            if (enumValues.hasMoreElements()) {
              // If attribute has more than one value, return false.
              logger.warn("nameDatasetDodsAttrib(): attribute has multiple values, only using first value <" + newDatasetName + ">");
              dataset.setName(newDatasetName);
              return (true);
            }
            // Desired attribute is a string and contains only one value,
            // set the dataset name and return true.
            logger.debug("nameDatasetDodsAttrib(): setting dataset name to <{}>.", newDatasetName);
            dataset.setName(newDatasetName);
            return (true);
          } else {
            // If attribute has no values, return false.
            logger.debug("nameDatasetDodsAttrib(): attribute has no value");
            return (false);
          }
        } else {
          // Desired attribute is not a string, return false.
          logger.debug("nameDatasetDodsAttrib(): attribute value is not a string.");
          return (false);
        }
      } else {
        // No such attribute container, return false.
        logger.debug("nameDatasetDodsAttrib(): attribute container does not exist.");
        return (false);
      }

    } catch (NoSuchAttributeException e) {
        // No such attribute container, return false.
        logger.debug("nameDatasetDodsAttrib(): attribute container does not exist.");
        return (false);
    }
  } */
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.bufr;

import ucar.unidata.io.RandomAccessFile;

import javax.annotation.concurrent.Immutable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Represents  Section 3 of a BUFR message.
 * @author caron
 * @since May 10, 2008
 */
@Immutable
public class BufrDataDescriptionSection {

  /**
   * Offset to start of BufrDataDescriptionSection.
   */
  private final long offset;

  /**
   * Number of data sets.
   */
  private final int ndatasets;

  /**
   * data type (observed or compressed).
   */
  private final int datatype;

  /**
   * List of data set descriptors.
   */
  private final List<Short> descriptors = new ArrayList<Short>();

  /**
   * Constructs a BufrDataDescriptionSection object by reading section 3 from a BUFR file.
   *
   * @param raf RandomAccessFile, position must be on a BUFR section 3
   * @throws IOException on read error
   */
  public BufrDataDescriptionSection(RandomAccessFile raf) throws IOException {
    offset = raf.getFilePointer();
    int length = BufrNumbers.uint3(raf);
    long EOS = offset + length;

    // reserved byte
    raf.read();

    // octets 5-6 number of datasets
    ndatasets = BufrNumbers.uint2(raf);

    // octet 7 data type bit 2 is for compressed data 192 or 64,
    // non-compressed data is 0 or 128
    datatype = raf.read();

    // get descriptors
    int ndesc = (length - 7) / 2;
    for (int i = 0; i < ndesc; i++) {
      int ch1 = raf.read();
      int ch2 = raf.read();
      short fxy = (short) ((ch1 << 8) + (ch2));
      descriptors.add(fxy);
    }

    // reset for any offset discrepancies
    raf.seek(EOS);
  }

  /**
   * Offset to the beginning of BufrDataDescriptionSection.
   *
   * @return offset in bytes of BUFR record
   */
  public final long getOffset() {
    return offset;
  }

  /**
   * Number of data sets in this record.
   *
   * @return datasets
   */
  public final int getNumberDatasets() {
    return ndatasets;
  }

  /**
   * Data type (compressed or non-compressed).
   *
   * @return datatype
   */
  public final int getDataType() {
    return datatype;
  }

  /**
   * Observation data
   *
   * @return true if observation data
   */
  public boolean isObserved() {
    return (datatype & 0x80) != 0;
  }

  /**
   * Is data compressed?
   *
   * @return true if data is compressed
   */
  public boolean isCompressed() {
    return (datatype & 0x40) != 0;
  }

  /**
   * get list of data descriptors as Shorts
   *
   * @return descriptors as List<Short>
   */
  public final List<Short> getDataDescriptors() {
    return descriptors;
  }

  /**
   * get list of data descriptors as Strings
   *
   * @return descriptors as List<String>
   */
  public final List<String> getDescriptors() {
    List<String> desc = new ArrayList<String>();
    for (short fxy : descriptors)
      desc.add(Descriptor.makeString(fxy));
    return desc;
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
//
// Filename: Utm_To_Gdc_Converter.java
//
// Author: Dan Toms, SRI International
//
// Package: GeoTransform http://www.ai.sri.com/geotransform/
//
// Acknowledgements:
// The algorithms used in the package were created by Ralph Toms and
// first appeared as part of the SEDRIS Coordinate Transformation API.
// These were subsequently modified for this package. This package is
// not part of the SEDRIS project, and the Java code written for this
// package has not been certified or tested for correctness by NIMA.
//
// License:
// The contents of this file are subject to GeoTransform License Agreement
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
// http://www.ai.sri.com/geotransformtest/license.html
//
// Software distributed under the License is distributed on an "AS IS"
// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
// the License for the specific language governing rights and limitations
// under the License.
//
// Portions are Copyright (c) SRI International, 1998.
//

package ucar.unidata.geoloc.projection;

import ucar.unidata.geoloc.EarthEllipsoid;
import ucar.unidata.geoloc.LatLonPoint;

import ucar.unidata.geoloc.LatLonPointImpl;

import java.lang.*;

/**
 * Class: Utm_To_Gdc_Converter
 *

 * Converts UTM coordinate(s) to GDC.
 *

 * This class provides the capability to convert from
 * Universal Transverse Mercator (UTM) coordinates to
 * geodetic (GDC), i.e. lat/long.
 * This is a direct conversion.
 *
 * @author Dan Toms, SRI International
 *

 *         modified JCaron 01/2005
 *

 *
 turn static methods into object methods, to make thread-safe
 *
 rename methods to follow upper/lower case conventions
 *
 add convenience methods for ucar.unidata.geoloc.Projection
 *
 longitude must be in range +=180.
 *

 *

 *         random testing shows:
 *         avg error x= 0.4 y=0.06 meters
 *         but sometimes x error can be as high as 15 meters
 *         where err = abs(x - inverse(f(x)))
 *

 *         timing: inverse(f(x)) takes 2 - 3 microseconds.
 */

class Utm_To_Gdc_Converter {

  static final double DEGREES_PER_RADIAN = 57.2957795130823208768;

  private double A; // major axis
  private double F; // flattening
  private double Eps2, Eps25, Con2, EF, Epsp2, Con6, Con24, Con120, Con720, polx2b, polx3b, polx4b, polx5b, conap;

  private int zone;

  private boolean hemisphere_north;

  /**
   * Constructor using a, f.
   *
   * @param a                the semi-major axis (meters) for the ellipsoid
   * @param f                the inverse flattening for the ellipsoid
   * @param zone             the UTM zone number (1..60)
   * @param hemisphere_north true if the UTM coordinate is in the northern hemisphere
   */
  public Utm_To_Gdc_Converter(double a, double f, int zone, boolean hemisphere_north) {
    init(a, f, zone, hemisphere_north);
  }

  /**
   * Default contructor uses WGS 84 ellipsoid
   *
   * @param zone             the UTM zone number (1..60)
   * @param hemisphere_north true if the UTM coordinate is in the northern hemisphere
   */
  public Utm_To_Gdc_Converter(int zone, boolean hemisphere_north) {
    this(EarthEllipsoid.WGS84, zone, hemisphere_north);  // default to wgs 84
  }

  /*
  * Constructor with ellipsoid.
  * @param E an Ellipsoid instance for the ellipsoid, e.g. WE_Ellipsoid
  * @param zone             the UTM zone number (1..60)
  * @param hemisphere_north true if the UTM coordinate is in the northern hemisphere
  */

  /**
   * Constructor using given ellipse
   *
   * @param ellipse          use this ellipse.
  * @param zone             the UTM zone number (1..60)
  * @param hemisphere_north true if the UTM coordinate is in the northern hemisphere
   */
  public Utm_To_Gdc_Converter(EarthEllipsoid ellipse, int zone, boolean hemisphere_north) {
    init(ellipse.getMajor(), 1.0 / ellipse.getFlattening(), zone, hemisphere_north);
  }

  /**
   * initialize
   *
   * @param a                major axis
   * @param f                inverse flattening
   * @param zone             UTM zone
   * @param hemisphere_north is in northern hemisphere
   */
  private void init(double a, double f, int zone, boolean hemisphere_north) {
    A = a;
    F = 1.0 / f; // F is flattening
    this.zone = zone;
    this.hemisphere_north = hemisphere_north;

    //  Create the ERM constants.
    Eps2 = (F) * (2.0 - F);
    Eps25 = .25 * (Eps2);
    EF = F / (2.0 - F);
    Con2 = 2 / (1.0 - Eps2);
    Con6 = .166666666666667;
    Con24 = 4 * .0416666666666667 / (1 - Eps2);
    Con120 = .00833333333333333;
    Con720 = 4 * .00138888888888888 / (1 - Eps2);
    double polx1a = 1.0 - Eps2 / 4.0 - 3.0 / 64.0 * Math.pow(Eps2, 2)
        - 5.0 / 256.0 * Math.pow(Eps2, 3)
        - 175.0 / 16384.0 * Math.pow(Eps2, 4);

    conap = A * polx1a;

    double polx2a = 3.0 / 2.0 * EF - 27.0 / 32.0 * Math.pow(EF, 3);

    double polx4a = 21.0 / 16.0 * Math.pow(EF, 2)
        - 55.0 / 32.0 * Math.pow(EF, 4);

    double polx6a = 151.0 / 96.0 * Math.pow(EF, 3);

    double polx8a = 1097.0 / 512.0 * Math.pow(EF, 4);

    polx2b = polx2a * 2.0 + polx4a * 4.0 + polx6a * 6.0 + polx8a * 8.0;

    polx3b = polx4a * -8.0 - polx6a * 32.0 - 80.0 * polx8a;
    polx4b = polx6a * 32.0 + 192.0 * polx8a;
    polx5b = -128.0 * polx8a;
  }

  public double getA() {
    return A;
  }

  public double getF() {
    return F;
  }

  public int getZone() {
    return zone;
  }

  public boolean isNorth() {
    return hemisphere_north;
  }

  /**
   * @param x      the UTM easting coordinate (meters)
   * @param y      the UTM northing coordinate (meters)
   * @param latlon put result here
   * @return LatLonPoint
   */
  public LatLonPoint projToLatLon(double x, double y, LatLonPointImpl latlon) {

    double source_x, source_y, u, su, cu, su2, xlon0, temp, phi1;
    double sp, sp2, cp, cp2, tp, tp2, eta2, top, rn, b3, b4, b5, b6, d1,
        d2;

    source_x = x * 1000.0;  // wants meters
    source_x = (source_x - 500000.0) / .9996;

    if (hemisphere_north) {
      source_y = y * 1000.0 / .9996;
    } else {
      source_y = (y * 1000.0 - 1.0E7) / .9996;
    }

    u = source_y / conap;

    /* TEST U TO SEE IF AT POLES */
    su = Math.sin(u);
    cu = Math.cos(u);
    su2 = su * su;

    xlon0 = (6.0 * ((double) zone) - 183.0) / DEGREES_PER_RADIAN;
    temp = polx2b + su2 * (polx3b + su2 * (polx4b + su2 * polx5b));
    phi1 = u + su * cu * temp;

    sp = Math.sin(phi1);
    cp = Math.cos(phi1);
    tp = sp / cp;
    tp2 = tp * tp;
    sp2 = sp * sp;
    cp2 = cp * cp;
    eta2 = Epsp2 * cp2;

    top = .25 - (sp2 * (Eps2 / 4));

    /* inline sq root*/
    rn = A / ((.25 - Eps25 * sp2 + .9999944354799 / 4)
        + (.25 - Eps25 * sp2)
        / (.25 - Eps25 * sp2 + .9999944354799 / 4));

    b3 = 1.0 + tp2 + tp2 + eta2;
    b4 = 5 + tp2 * (3 - 9 * eta2) + eta2 * (1 - 4 * eta2);
    b5 = 5 + tp2 * (tp2 * 24.0 + 28.0);
    b5 += eta2 * (tp2 * 8.0 + 6.0);
    b6 = 46.0 - 3.0 * eta2 + tp2 * (-252.0 - tp2 * 90.0);
    b6 = eta2 * (b6 + eta2 * tp2 * (tp2 * 225.0 - 66.0));
    b6 += 61.0 + tp2 * (tp2 * 45.0 + 90.0);

    d1 = source_x / rn;
    d2 = d1 * d1;

    double latitude = phi1
        - tp * top
        * (d2 * (Con2
        + d2 * ((-Con24) * b4
        + d2 * Con720 * b6)));
    double longitude = xlon0
        + d1 * (1.0
        + d2 * (-Con6 * b3
        + d2 * Con120 * b5)) / cp;

    latlon.setLatitude(latitude * DEGREES_PER_RADIAN);
    latlon.setLongitude(longitude * DEGREES_PER_RADIAN);
    return latlon;
  }

  /**
   * _more_
   *
   * @param from _more_
   * @param to   _more_
   * @return _more_
   */
  public float[][] projToLatLon(float[][] from, float[][] to) {

    double source_x, source_y, u, su, cu, su2, temp, phi1;
    double sp, sp2, cp, cp2, tp, tp2, eta2, top, rn, b3, b4, b5, b6, d1,
        d2;

    double xlon0 = (6.0 * ((double) zone) - 183.0) / DEGREES_PER_RADIAN;

    for (int i = 0; i < from[0].length; i++) {
      source_x = from[0][i] * 1000.0;  // wants meters
      source_x = (source_x - 500000.0) / .9996;

      if (hemisphere_north) {
        source_y = from[1][i] * 1000.0 / .9996;
      } else {
        source_y = (from[1][i] * 1000.0 - 1.0E7) / .9996;
      }

      u = source_y / conap;
      su = Math.sin(u);
      cu = Math.cos(u);
      su2 = su * su;

      temp = polx2b + su2 * (polx3b + su2 * (polx4b + su2 * polx5b));
      phi1 = u + su * cu * temp;

      sp = Math.sin(phi1);
      cp = Math.cos(phi1);
      tp = sp / cp;
      tp2 = tp * tp;
      sp2 = sp * sp;
      cp2 = cp * cp;
      eta2 = Epsp2 * cp2;

      top = .25 - (sp2 * (Eps2 / 4));

      /* inline sq root*/
      rn = A / ((.25 - Eps25 * sp2 + .9999944354799 / 4)
          + (.25 - Eps25 * sp2)
          / (.25 - Eps25 * sp2 + .9999944354799 / 4));

      b3 = 1.0 + tp2 + tp2 + eta2;
      b4 = 5 + tp2 * (3 - 9 * eta2) + eta2 * (1 - 4 * eta2);
      b5 = 5 + tp2 * (tp2 * 24.0 + 28.0);
      b5 += eta2 * (tp2 * 8.0 + 6.0);
      b6 = 46.0 - 3.0 * eta2 + tp2 * (-252.0 - tp2 * 90.0);
      b6 = eta2 * (b6 + eta2 * tp2 * (tp2 * 225.0 - 66.0));
      b6 += 61.0 + tp2 * (tp2 * 45.0 + 90.0);

      d1 = source_x / rn;
      d2 = d1 * d1;

      double latitude = phi1
          - tp * top
          * (d2 * (Con2
          + d2 * ((-Con24) * b4
          + d2 * Con720 * b6)));
      double longitude = xlon0
          + d1 * (1.0
          + d2 * (-Con6 * b3
          + d2 * Con120 * b5)) / cp;

      to[0][i] = (float) (latitude * DEGREES_PER_RADIAN);
      to[1][i] = (float) (longitude * DEGREES_PER_RADIAN);
    }

    return to;
  }

  /**
   * _more_
   *
   * @param from _more_
   * @param to   _more_
   * @return _more_
   */
  public double[][] projToLatLon(double[][] from, double[][] to) {

    double source_x, source_y, u, su, cu, su2, temp, phi1;
    double sp, sp2, cp, cp2, tp, tp2, eta2, top, rn, b3, b4, b5, b6, d1,
        d2;

    double xlon0 = (6.0 * ((double) zone) - 183.0) / DEGREES_PER_RADIAN;

    for (int i = 0; i < from[0].length; i++) {
      source_x = from[0][i] * 1000.0;  // wants meters
      source_x = (source_x - 500000.0) / .9996;

      if (hemisphere_north) {
        source_y = from[1][i] * 1000.0 / .9996;
      } else {
        source_y = (from[1][i] * 1000.0 - 1.0E7) / .9996;
      }

      u = source_y / conap;
      su = Math.sin(u);
      cu = Math.cos(u);
      su2 = su * su;

      temp = polx2b + su2 * (polx3b + su2 * (polx4b + su2 * polx5b));
      phi1 = u + su * cu * temp;

      sp = Math.sin(phi1);
      cp = Math.cos(phi1);
      tp = sp / cp;
      tp2 = tp * tp;
      sp2 = sp * sp;
      cp2 = cp * cp;
      eta2 = Epsp2 * cp2;

      top = .25 - (sp2 * (Eps2 / 4));

      /* inline sq root*/
      rn = A / ((.25 - Eps25 * sp2 + .9999944354799 / 4)
          + (.25 - Eps25 * sp2)
          / (.25 - Eps25 * sp2 + .9999944354799 / 4));

      b3 = 1.0 + tp2 + tp2 + eta2;
      b4 = 5 + tp2 * (3 - 9 * eta2) + eta2 * (1 - 4 * eta2);
      b5 = 5 + tp2 * (tp2 * 24.0 + 28.0);
      b5 += eta2 * (tp2 * 8.0 + 6.0);
      b6 = 46.0 - 3.0 * eta2 + tp2 * (-252.0 - tp2 * 90.0);
      b6 = eta2 * (b6 + eta2 * tp2 * (tp2 * 225.0 - 66.0));
      b6 += 61.0 + tp2 * (tp2 * 45.0 + 90.0);

      d1 = source_x / rn;
      d2 = d1 * d1;

      double latitude = phi1
          - tp * top
          * (d2 * (Con2
          + d2 * ((-Con24) * b4
          + d2 * Con720 * b6)));
      double longitude = xlon0
          + d1 * (1.0
          + d2 * (-Con6 * b3
          + d2 * Con120 * b5)) / cp;

      to[0][i] = (latitude * DEGREES_PER_RADIAN);
      to[1][i] = (longitude * DEGREES_PER_RADIAN);
    }

    return to;
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.logs;

import ucar.nc2.util.EscapeStrings;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.regex.*;
import java.util.Formatter;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.File;

/**
 * Read TDS access logs
 *
 * @author caron
 * @since Apr 10, 2008
 */
public class ServletLogParser implements LogReader.LogParser {
  static public class ServletLog extends LogReader.Log {
    long reqTime;
    long reqSeq;
    String level, where;
    public StringBuilder extra;
    boolean isDone, isStart;
    int nthreads;

    public long getReqTime() {
      return reqTime;
    }

    public long getReqSeq() {
      return reqSeq;
    }

    public String getLevel() {
      return level;
    }

    public boolean isExtra() {
      return extra != null;
    }

    public boolean isDone() {
      return isDone;
    }

    public boolean isStart() {
      return isStart;
    }

    public int getNthreads() {
      return nthreads;
    }

    public void setNthreads(int nthreads) {
      this.nthreads = nthreads;
    }

    public String toString() {
      Formatter f = new Formatter();
      f.format("%s [%d] [%d] %s %s: ", getDate(), reqTime, reqSeq, level, where);

      if (isStart)
        f.format(" (%s) %s %n", ip, getPath());
      else if (isDone)
        f.format(" %d %d %d %n", returnCode, sizeBytes, msecs);

      if (extra != null)
        f.format(" %s", extra);

      return f.toString();
    }

    void addExtra(String s) {
      if (extra == null) extra = new StringBuilder(300);
      extra.append(s);
      extra.append("\n");
    }

  }

  ///////////////////////////////////////////////////////
  // log reading

  // sample
  // 128.117.140.75 - - [02/May/2008:00:46:26 -0600] "HEAD /thredds/dodsC/model/NCEP/DGEX/CONUS_12km/DGEX_CONUS_12km_20080501_1800.grib2.dds HTTP/1.1" 200 - "null" "Java/1.6.0_05" 21
  //           Pattern.compile("^(\\d+\\.\\d+\\.\\d+\\.\\d+) - (.*) \\[(.*)\\] \"(.*)\" (\\d+) ([\\-\\d]+) \"(.*)\" \"(.*)\" (\\d+)");

  // 2009-03-10T16:08:55.184 -0600 [  16621850][  162233] INFO  - thredds.server.opendap.NcDODSServlet - Remote host: 128.117.140.71 - Request: "GET /thredds/dodsC/model/NCEP/NAM/CONUS_80km/NAM_CONUS_80km_20090309_0000.grib1.dods?Geopotential_height%5B7:1:7%5D%5B0:10:10%5D%5B0:1:64%5D%5B0:1:92%5D HTTP/1.1"
  // 2009-03-10T16:08:55.218 -0600 [  16621884][  162233] INFO  - thredds.server.opendap.NcDODSServlet - Request Completed - 200 - -1 - 34
  //
  /*
   1 2009-03-10T16:08:54.617 -0600
   2   16621283
   3   162230
   4 INFO
   5 thredds.server.opendap.NcDODSServlet
   6 Request Completed - 200 - -1 - 47
   */

  static private final Pattern donePattern = Pattern.compile("^Request Completed - (.*) - (.*) - (.*)");
  static private final Pattern startPattern = Pattern.compile("^Remote host: ([^-]+) - Request: \"(\\w+) (.*) (.*)");
  static private final Pattern commonPattern = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ [+-]\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - (.*)");

  /*
  Difficult thing is to return the extra line assocated with the previous good log
  We do this by not returning until we get a match on the next log. We have to rewind.
   */
  public LogReader.Log nextLog(BufferedReader dataIS) throws IOException {

    ServletLog log = new ServletLog();
    boolean haveLog = false;

    while (true) {
      dataIS.mark(20 * 1000); // track where we are
      String line = dataIS.readLine();
      if (line == null) {
        return haveLog ? log : null;
      }
      // if (count++ < limit) System.out.println("\n" + line);                      nTest reqe

      try {
        Matcher m = commonPattern.matcher(line);
        if (m.matches()) {
          if (haveLog) { // have a log, next one matches, proceed
            try {
              dataIS.reset();
              return log;

            } catch (Throwable t) {
              System.out.println("Cant reset " + line);
            }
          }
          haveLog = true; // next match will return the current log

          log.date = convertDate( m.group(1));
          log.reqTime = parseLong(m.group(2));
          log.reqSeq = parseLong(m.group(3));
          log.level = m.group(4).intern();
          log.where = m.group(5);

          String rest = m.group(6);
          if (rest.contains("Request Completed")) {
            int pos = rest.indexOf("Request Completed");
            Matcher m2 = donePattern.matcher(rest.substring(pos));
            if (m2.matches()) {
              log.returnCode = parse(m2.group(1));
              log.sizeBytes = parseLong(m2.group(2));
              log.msecs = parseLong(m2.group(3));
              log.isDone = true;

            } else {
              System.out.println("Cant parse donePattern= " + rest);
              System.out.println(" line= " + line);
              log.addExtra(rest);
            }

          } else if (rest.contains("Remote host")) {
            int pos = rest.indexOf("Remote host");
            Matcher m2 = startPattern.matcher(rest.substring(pos));
            if (m2.matches()) {
              log.ip = m2.group(1);
              log.verb = m2.group(2).intern();
              log.path = EscapeStrings.urlDecode(m2.group(3)); //old  URLDecoder.decode(m2.group(3));
              if (m2.groupCount() > 4)
                log.http = m2.group(4).intern();
              log.isStart = true;

            } else {
              System.out.println("Cant parse startPattern= " + rest);
              System.out.println(" line= " + line);
              log.addExtra(rest);
            }

          } else { // a non-start, non-done log
            log.addExtra(rest);
          }

        } else { // the true extra line
          //System.out.println("No match on " + line);
          log.addExtra(line);
        }

      } catch (Exception e) {
        e.printStackTrace();
        System.out.println("Cant parse " + line);
        log.addExtra(line);
      }
    }

  }

  // 2010-04-21T13:05:22.006 -0600
  private SimpleDateFormat df = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
  private long convertDate(String accessDateFormat) {
    // 30/Sep/2009:23:50:47 -0600
    try {
      Date d = df.parse(accessDateFormat);
      return d.getTime();
    } catch (Throwable t) {
      System.out.printf("Bad date format = %s err = %s%n", accessDateFormat, t.getMessage());
    }
    return -1;
  }

  private int parse(String s) {
    s = s.trim();
    if (s.isEmpty()) return 0;
    if (s.equals("-")) return 0;
    return Integer.parseInt(s);
  }

  private long parseLong(String s) {
    s = s.trim();
    if (s.isEmpty()) return 0;
    if (s.equals("-")) return 0;
    return Long.parseLong(s);
  }

  static class MyLogFilter implements LogReader.LogFilter {
    int count = 0;
    int limit;

    MyLogFilter(int limit) {
      this.limit = limit;
    }

    public boolean pass(LogReader.Log log) {
      return (limit < 0) || count++ < limit;
    }
  }

  public static void main1(String args[]) throws IOException {
    // test
    final LogReader reader = new LogReader(new ServletLogParser());

    long startElapsed = System.nanoTime();
    LogReader.Stats stats = new LogReader.Stats();

    reader.readAll(new File("D:/mlode/logs/servlet/"), null, new LogReader.Closure() {
      public void process(LogReader.Log log) throws IOException {
        //if (count < limit) System.out.printf(" %s %n", log);
      }
    }, new MyLogFilter(-1), stats);

    long elapsedTime = System.nanoTime() - startElapsed;
    System.out.printf(" total= %d passed=%d%n", stats.total, stats.passed);
    System.out.printf(" elapsed=%d secs%n", elapsedTime / (1000 * 1000 * 1000));
  }

  public static void main2(String args[]) {
    //           1                                2           3       4       5                                                   6                           7   8                                                                                9
    String s1 = "2009-03-10T16:08:55.184 -0600 [  16621850][  162233] INFO  - thredds.server.opendap.NcDODSServlet - Remote host: 128.117.140.71 - Request: \"GET /thredds/dodsC/model/NCEP/NAM/CONUS_80km/NAM_CONUS_80km_20090309_0000.grib1.dods?Geopotential_height%5B7:1:7%5D%5B0:10:10%5D%5B0:1:64%5D%5B0:1:92%5D HTTP/1.1\"";
    //                             1                                              2       3      4             5                      6                    7      8    9
    Pattern p1 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - Remote host: ([^-]+) - Request: \"(\\w+) (.*) (.*)");

    show(s1, p1);

    //           1                                2           3       4       5                                                           6    7    8                                                                                9
    String s2 = "2009-03-10T16:08:54.617 -0600 [  16621283][  162230] INFO  - thredds.server.opendap.NcDODSServlet - Request Completed - 200 - -1 - 47";
    //                             1                                               2       3     4              5                            6        7       8                                                                                9
    Pattern p2 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - Request Completed - (\\d+) - (.*) - (.*)");
//  Pattern p1 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - Remote host: ([^-]+) - Request: \"(\\w+) (.*) (.*)");

    show(s2, p2);

//  Pattern p2 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - Request Completed - (\\d+) - (.*) - (.*)");
//  Pattern p1 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - Remote host: ([^-]+) - Request: \"(\\w+) (.*) (.*)");
    Pattern p3 = Pattern.compile("^(\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+ -\\d+) \\[(.*)]\\[(.*)] (\\w+)[\\s]+- ([^-]+) - (.*)");

    show(s1, p3);
    show(s2, p3);

  }

  public static void main(String args[]) {

      //           1                                 2            3       4     5                                          6                           7   8                                                                                9
    String test = "2009-04-03T21:48:54.000 -0600 [   4432114][    1184] INFO  - thredds.servlet.ServletUtil - Remote host: 128.117.140.75 - Request: \"GET /thredds/dodsC/fmrc/NCEP/NAM/CONUS_20km/surface/forecast/NCEP-NAM-CONUS_20km-surface_ConstantForecast_2009-04-04T15:00:00Z.dods?time,time_run,time_offset,Lambert_Conformal,pressure_difference_layer_bounds,depth_below_surface_layer_bounds,pressure_difference_layer1_bounds,depth_below_surface_layer1_bounds,depth_below_surface_layer2_bounds,pressure_difference_layer2_bounds,pressure_difference_layer3_bounds,pressure_difference_layer4_bounds,pressure_difference_layer5_bounds,height_above_ground_layer1_bounds,pressure_layer_bounds,height_above_ground_layer_bounds,y,x,height_above_ground1,height_above_ground2,pressure_difference_layer,depth_below_surface_layer,pressure_difference_layer1,depth_below_surface_layer1,depth_below_surface_layer2,height_above_ground,pressure_difference_layer2,height_above_ground3,pressure,depth_below_surface,pressure1,pressure_difference_layer3,pressure_difference_layer4,pressure_difference_layer5,height_above_ground_layer1,pressure2,pressure_layer,height_above_ground_layer HTTP/1.1\"";
    show(test, commonPattern);

    String test2 = "Remote host: 128.117.140.75 - Request: \"GET /thredds/dodsC/fmrc/NCEP/NAM/CONUS_20km/surface/forecast/NCEP-NAM-CONUS_20km-surface_ConstantForecast_2009-04-04T15:00:00Z.dods?time,time_run,time_offset,Lambert_Conformal,pressure_difference_layer_bounds,depth_below_surface_layer_bounds,pressure_difference_layer1_bounds,depth_below_surface_layer1_bounds,depth_below_surface_layer2_bounds,pressure_difference_layer2_bounds,pressure_difference_layer3_bounds,pressure_difference_layer4_bounds,pressure_difference_layer5_bounds,height_above_ground_layer1_bounds,pressure_layer_bounds,height_above_ground_layer_bounds,y,x,height_above_ground1,height_above_ground2,pressure_difference_layer,depth_below_surface_layer,pressure_difference_layer1,depth_below_surface_layer1,depth_below_surface_layer2,height_above_ground,pressure_difference_layer2,height_above_ground3,pressure,depth_below_surface,pressure1,pressure_difference_layer3,pressure_difference_layer4,pressure_difference_layer5,height_above_ground_layer1,pressure2,pressure_layer,height_above_ground_layer HTTP/1.1\"";
    show(test2, startPattern);

  }

  private static void show(String s, Pattern p) {
    System.out.println("==============================");
    Matcher m = p.matcher(s);
    System.out.printf(" match against %s = %s %n", m, m.matches());
    if (!m.matches()) return;
    for (int i = 1; i <= m.groupCount(); i++)
      System.out.println(" " + i + " " + m.group(i));

  }
}