/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.catalog.parser.jdom;

import thredds.featurecollection.FeatureCollectionConfig;
import thredds.util.PathAliasReplacement;
import thredds.catalog.*;
import thredds.crawlabledataset.*;
import thredds.crawlabledataset.sorter.LexigraphicByNameSorter;
import thredds.crawlabledataset.filter.*;
import thredds.cataloggen.ProxyDatasetHandler;
import thredds.cataloggen.DatasetEnhancer;
import thredds.cataloggen.CatalogRefExpander;
import thredds.cataloggen.datasetenhancer.RegExpAndDurationTimeCoverageEnhancer;
import thredds.cataloggen.inserter.SimpleLatestProxyDsHandler;
import thredds.cataloggen.inserter.LatestCompleteProxyDsHandler;

import org.jdom2.*;
import org.jdom2.input.*;
import org.jdom2.output.*;

import java.io.*;
import java.lang.reflect.Method;
import java.net.*;
import java.util.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;

import ucar.nc2.constants.CDM;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.units.TimeDuration;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;

/**
 * Inventory Catalog parser, version 1.0.
 * Reads InvCatalog.xml files, constructs object representation.
 *
 * @author John Caron
 */

public class InvCatalogFactory10 implements InvCatalogConvertIF, MetadataConverterIF {
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InvCatalogFactory10.class);

  static public final Namespace defNS = Namespace.getNamespace(XMLEntityResolver.CATALOG_NAMESPACE_10);
  static public final Namespace xlinkNS = Namespace.getNamespace("xlink", XMLEntityResolver.XLINK_NAMESPACE);
  static public final Namespace ncmlNS = Namespace.getNamespace("ncml", XMLEntityResolver.NJ22_NAMESPACE);

  static private boolean useBytesForDataSize = false;

  static public void useBytesForDataSize(boolean b) {
    useBytesForDataSize = b;
  }

  private InvCatalogFactory factory = null;

  private String version = "1.0.1";
  private boolean debugMetadataRead = false;

  private List<PathAliasReplacement> dataRootLocAliasExpanders = Collections.emptyList();

  public void setDataRootLocationAliasExpanders(List<PathAliasReplacement> dataRootLocAliasExpanders) {
    if (dataRootLocAliasExpanders == null)
      this.dataRootLocAliasExpanders = Collections.emptyList();
    else
      this.dataRootLocAliasExpanders = new ArrayList<>(dataRootLocAliasExpanders);
  }

  public List<PathAliasReplacement> getDataRootLocationAliasExpanders() {
    return Collections.unmodifiableList(this.dataRootLocAliasExpanders);
  }

  private String expandAliasForPath(String location) {
    for (PathAliasReplacement par : this.dataRootLocAliasExpanders) {
      if (par.containsPathAlias(location)) {
        return par.replacePathAlias(location);
      }
    }
    return location;
  }

  private String expandAliasForCollectionSpec(String location) {
    if (location == null) return null;
    for (PathAliasReplacement par : this.dataRootLocAliasExpanders) {
      String result = par.replaceIfMatch(location);
      if (result != null) return result;
    }
    return location;
  }

  public InvCatalogImpl parseXML(InvCatalogFactory fac, org.jdom2.Document jdomDoc, URI uri) {
    this.factory = fac;
    return readCatalog(jdomDoc.getRootElement(), uri);
  }

  private Map<MetadataType, MetadataConverterIF> metadataHash = new HashMap<>(10);

  public void registerMetadataConverter(MetadataType type, MetadataConverterIF converter) {
    metadataHash.put(type, converter);
  }

  public void setVersion(String version) {
    this.version = version;
  }

  /////////////////////////////////////////////////////////////////////////////

  protected InvAccessImpl readAccess(InvDatasetImpl dataset, Element accessElem) {
    String urlPath = accessElem.getAttributeValue("urlPath");
    String serviceName = accessElem.getAttributeValue("serviceName");
    String dataFormat = accessElem.getAttributeValue("dataFormat");

    return new InvAccessImpl(dataset, urlPath, serviceName, null, dataFormat, readDataSize(accessElem));
  }

  protected InvCatalogImpl readCatalog(Element catalogElem, URI docBaseURI) {
    String name = catalogElem.getAttributeValue("name");
    String catSpecifiedBaseURL = catalogElem.getAttributeValue("base");
    String expires = catalogElem.getAttributeValue("expires");
    String version = catalogElem.getAttributeValue("version");

    URI baseURI = docBaseURI;
    if (catSpecifiedBaseURL != null) {
      try {
        baseURI = new URI(catSpecifiedBaseURL);
      } catch (URISyntaxException e) {
        logger.debug("readCatalog(): bad catalog specified base URI <" + catSpecifiedBaseURL + ">: " + e.getMessage(), e);
        baseURI = docBaseURI;
      }
    }

    InvCatalogImpl catalog = new InvCatalogImpl(name, version, makeDateType(expires, null, null), baseURI);

    // read top-level services
    java.util.List<Element> sList = catalogElem.getChildren("service", defNS);
    for (Element e : sList) {
      InvService s = readService(e, baseURI);
      catalog.addService(s);
    }

    // read top-level properties
    java.util.List<Element> pList = catalogElem.getChildren("property", defNS);
    for (Element e : pList) {
      InvProperty s = readProperty(e);
      catalog.addProperty(s);
    }

    // read top-level dataroots
    java.util.List<Element> rootList = catalogElem.getChildren("datasetRoot", defNS);
    for (Element e : rootList) {
      DataRootConfig root = readDatasetRoot(e);
      catalog.addDatasetRoot(root);
    }

    // look for top-level dataset and catalogRefs elements (keep them in order)
    java.util.List<Element> allChildren = catalogElem.getChildren();
    for (Element e : allChildren) {
      if (e.getName().equals("dataset")) {
        catalog.addDataset(readDataset(catalog, null, e, baseURI));
      } else if (e.getName().equals("featureCollection")) {
        catalog.addDataset(readFeatureCollection(catalog, null, e, baseURI));
      } else if (e.getName().equals("datasetScan")) {
        catalog.addDataset(readDatasetScan(catalog, null, e, baseURI));
      } else if (e.getName().equals("catalogRef")) {
        catalog.addDataset(readCatalogRef(catalog, null, e, baseURI));
      }
    }

    return catalog;
  }

  protected InvCatalogRef readCatalogRef(InvCatalogImpl cat, InvDatasetImpl parent, Element catRefElem, URI baseURI) {
    String title = catRefElem.getAttributeValue("title", xlinkNS);
    if (title == null) title = catRefElem.getAttributeValue("name");
    String href = catRefElem.getAttributeValue("href", xlinkNS);
    String useRemCatSerStr = catRefElem.getAttributeValue("useRemoteCatalogService");
    Boolean useRemoteCatalogService = null;
    if (useRemCatSerStr != null) useRemoteCatalogService = Boolean.parseBoolean(useRemCatSerStr);
    InvCatalogRef catRef = new InvCatalogRef(parent, title, href, useRemoteCatalogService);
    readDatasetInfo(cat, catRef, catRefElem, baseURI);
    return catRef;
  }

  protected ThreddsMetadata.Contributor readContributor(Element elem) {
    if (elem == null) return null;
    return new ThreddsMetadata.Contributor(elem.getText(), elem.getAttributeValue("role"));
  }

  protected ThreddsMetadata.Vocab readControlledVocabulary(Element elem) {
    if (elem == null) return null;
    return new ThreddsMetadata.Vocab(elem.getText(), elem.getAttributeValue("vocabulary"));
  }

  // read a dataset element
  protected InvDatasetImpl readDataset(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {

    // deal with aliases
    String name = dsElem.getAttributeValue("name");
    String alias = dsElem.getAttributeValue("alias");
    if (alias != null) {
      InvDatasetImpl ds = (InvDatasetImpl) catalog.findDatasetByID(alias);
      if (ds == null) {
        factory.appendErr(" ** Parse error: dataset named " + name + " has illegal alias = " + alias + "\n");
        return null;
      }
      return new InvDatasetImplProxy(name, ds);
    }

    InvDatasetImpl dataset = new InvDatasetImpl(parent, name);
    readDatasetInfo(catalog, dataset, dsElem, base);

    if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + dataset.dump());
    return dataset;
  }

  protected void readDatasetInfo(InvCatalogImpl catalog, InvDatasetImpl dataset, Element dsElem, URI base) {
    // read attributes
    String authority = dsElem.getAttributeValue("authority");
    String collectionTypeName = dsElem.getAttributeValue("collectionType");
    String dataTypeName = dsElem.getAttributeValue("dataType");
    String harvest = dsElem.getAttributeValue("harvest");
    String id = dsElem.getAttributeValue("ID");
    String serviceName = dsElem.getAttributeValue("serviceName");
    String urlPath = dsElem.getAttributeValue("urlPath");
    String restrictAccess = dsElem.getAttributeValue("restrictAccess");

    FeatureType dataType = null;
    if (dataTypeName != null) {
      dataType = FeatureType.getType(dataTypeName.toUpperCase());
      if (dataType == null) {
        factory.appendWarning(" ** warning: non-standard data type = " + dataTypeName + "\n");
      }
    }

    if (dataType != null)
      dataset.setDataType(dataType);
    if (serviceName != null)
      dataset.setServiceName(serviceName);
    if (urlPath != null)
      dataset.setUrlPath(urlPath);

    if (authority != null) dataset.setAuthority(authority);
    if (id != null) dataset.setID(id);
    if (harvest != null) dataset.setHarvest(harvest.equalsIgnoreCase("true"));
    if (restrictAccess != null)
      dataset.setResourceControl(restrictAccess);

    if (collectionTypeName != null) {
      CollectionType collectionType = CollectionType.findType(collectionTypeName);
      if (collectionType == null) {
        collectionType = CollectionType.getType(collectionTypeName);
        factory.appendWarning(" ** warning: non-standard collection type = " + collectionTypeName + "\n");
      }
      dataset.setCollectionType(collectionType);
    }

    catalog.addDatasetByID(dataset); // need to do immed for alias processing

    // look for services
    java.util.List<Element> serviceList = dsElem.getChildren("service", defNS);
    for (Element curElem : serviceList) {
      InvService s = readService(curElem, base);
      dataset.addService(s);
    }

    // look for direct thredds metadata (not inherited)
    ThreddsMetadata tmg = dataset.getLocalMetadata();
    readThreddsMetadata(catalog, dataset, dsElem, tmg);

    // look for access elements
    java.util.List<Element> aList = dsElem.getChildren("access", defNS);
    for (Element e : aList) {
      InvAccessImpl a = readAccess(dataset, e);
      dataset.addAccess(a);
    }

    // look for ncml
    Element ncmlElem = dsElem.getChild("netcdf", ncmlNS);
    if (ncmlElem != null) {
      ncmlElem.detach();
      dataset.setNcmlElement(ncmlElem);
      // System.out.println(" found ncml= "+ncmlElem);
    }

    // look for nested dataset and catalogRefs elements (keep them in order)
    java.util.List<Element> allChildren = dsElem.getChildren();
    for (Element e : allChildren) {
      if (e.getName().equals("dataset")) {
        InvDatasetImpl ds = readDataset(catalog, dataset, e, base);
        if (ds != null)
          dataset.addDataset(ds);
      } else if (e.getName().equals("catalogRef")) {
        InvDatasetImpl ds = readCatalogRef(catalog, dataset, e, base);
        dataset.addDataset(ds);
      } else if (e.getName().equals("datasetScan")) {
        dataset.addDataset(readDatasetScan(catalog, dataset, e, base));
      } else if (e.getName().equals("featureCollection")) {
        InvDatasetImpl ds = readFeatureCollection(catalog, dataset, e, base);
        if (ds != null)
          dataset.addDataset(ds);
      }
    }
  }

  protected InvDatasetImpl readFeatureCollection(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {

    FeatureCollectionConfig config = FeatureCollectionReader.readFeatureCollection(dsElem);
    config.spec = expandAliasForCollectionSpec(config.spec);

    try {
      // InvDatasetFeatureCollection ds = InvDatasetFeatureCollection.factory(parent, config);
      Class c = this.getClass().getClassLoader().loadClass("thredds.catalog.InvDatasetFeatureCollection");
      Method m = c.getMethod("factory", InvDatasetImpl.class, FeatureCollectionConfig.class);
      InvDatasetImpl result = (InvDatasetImpl) m.invoke(null, parent, config);

      if (result == null) {
        logger.error("featureCollection " + config.collectionName + " has fatal error ");
        return null;
      }

      // regular dataset elements
      readDatasetInfo(catalog, result, dsElem, base);
      return result;

    } catch (Exception e) {
      logger.error("featureCollection " + config.collectionName + " has fatal error, skipping ", e);
      return null;
    }

  }

  /* protected InvDatasetImpl readFeatureCollection(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {

    FeatureCollectionConfig config = FeatureCollectionReader.readFeatureCollection(dsElem);
    config.spec = expandAliasForCollectionSpec(config.spec);

    try {
      InvDatasetFeatureCollection ds = InvDatasetFeatureCollection.factory(parent, config);
      if (ds == null) {
        logger.error("featureCollection " + config.collectionName + " has fatal error ");
        return null;
      }
      // regular dataset elements
      readDatasetInfo(catalog, ds, dsElem, base);
      return ds;

    } catch (Exception e) {
      logger.error("featureCollection " + config.collectionName + " has fatal error, skipping ", e);
      return null;
    }

  } */

  // read a dataset scan element
  protected InvDatasetScan readDatasetScan(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
    InvDatasetScan datasetScan;

    if (dsElem.getAttributeValue("dirLocation") == null) {
      if (dsElem.getAttributeValue("location") == null) {
        logger.error("readDatasetScan(): datasetScan has neither a \"location\" nor a \"dirLocation\" attribute.");
        datasetScan = null;
      } else {
        return readDatasetScanNew(catalog, parent, dsElem, base);
      }
    } else {
      String name = dsElem.getAttributeValue("name");
      factory.appendWarning("**Warning: Dataset " + name + " using old form of DatasetScan (dirLocation instead of location)\n");

      String path = dsElem.getAttributeValue("path");

      String scanDir = expandAliasForPath(dsElem.getAttributeValue("dirLocation"));
      String filter = dsElem.getAttributeValue("filter");
      String addDatasetSizeString = dsElem.getAttributeValue("addDatasetSize");
      String addLatest = dsElem.getAttributeValue("addLatest");
      String sortOrderIncreasingString = dsElem.getAttributeValue("sortOrderIncreasing");
      boolean sortOrderIncreasing = false;
      if (sortOrderIncreasingString != null)
        if (sortOrderIncreasingString.equalsIgnoreCase("true"))
          sortOrderIncreasing = true;
      boolean addDatasetSize = true;
      if (addDatasetSizeString != null)
        if (addDatasetSizeString.equalsIgnoreCase("false"))
          addDatasetSize = false;

      if (path != null) {
        if (path.charAt(0) == '/') path = path.substring(1);
        int last = path.length() - 1;
        if (path.charAt(last) == '/') path = path.substring(0, last);
      }

      if (scanDir != null) {
        int last = scanDir.length() - 1;
        if (scanDir.charAt(last) != '/') scanDir = scanDir + '/';
      }

      Element atcElem = dsElem.getChild("addTimeCoverage", defNS);
      String dsNameMatchPattern = null;
      String startTimeSubstitutionPattern = null;
      String duration = null;
      if (atcElem != null) {
        dsNameMatchPattern = atcElem.getAttributeValue("datasetNameMatchPattern");
        startTimeSubstitutionPattern = atcElem.getAttributeValue("startTimeSubstitutionPattern");
        duration = atcElem.getAttributeValue("duration");
      }

      try {
        datasetScan = new InvDatasetScan(catalog, parent, name, path, scanDir, filter, addDatasetSize, addLatest, sortOrderIncreasing,
                dsNameMatchPattern, startTimeSubstitutionPattern, duration);
        readDatasetInfo(catalog, datasetScan, dsElem, base);
        if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + datasetScan.dump());

      } catch (Exception e) {
        logger.error("Reading DatasetScan", e);
        datasetScan = null;
      }
    }

    return datasetScan;
  }

  protected InvDatasetScan readDatasetScanNew(InvCatalogImpl catalog, InvDatasetImpl parent, Element dsElem, URI base) {
    String name = dsElem.getAttributeValue("name");
    String path = dsElem.getAttributeValue("path");

    String locationOrg = dsElem.getAttributeValue("location");
    String scanDir = expandAliasForPath(locationOrg);
    if (scanDir.startsWith("${")) {
      scanDir = expandAliasForPath(locationOrg);  // debugging
    }

    // Read datasetConfig element
    String configClassName = null;
    Object configObj = null;
    Element dsConfigElem = dsElem.getChild("crawlableDatasetImpl", defNS);
    if (dsConfigElem != null) {
      configClassName = dsConfigElem.getAttributeValue("className");
      List children = dsConfigElem.getChildren();
      if (children.size() == 1) {
        configObj = children.get(0);
      } else if (children.size() != 0) {
        logger.warn("readDatasetScanNew(): content of datasetConfig element not a single element, using first element.");
        configObj = children.get(0);
      } else {
        logger.debug("readDatasetScanNew(): datasetConfig element has no children.");
        configObj = null;
      }
    }

    // Read filter element
    Element filterElem = dsElem.getChild("filter", defNS);
    CrawlableDatasetFilter filter = null;
    if (filterElem != null)
      filter = readDatasetScanFilter(filterElem);

    // Read identifier element
    Element identifierElem = dsElem.getChild("addID", defNS);
    CrawlableDatasetLabeler identifier = null;
    if (identifierElem != null) {
      identifier = readDatasetScanIdentifier(identifierElem);
    }

    // Read namer element
    Element namerElem = dsElem.getChild("namer", defNS);
    CrawlableDatasetLabeler namer = null;
    if (namerElem != null) {
      namer = readDatasetScanNamer(namerElem);
    }

    // Read sort element
    Element sorterElem = dsElem.getChild("sort", defNS);
    // By default, sort in decreasing lexigraphic order.
    CrawlableDatasetSorter sorter = new LexigraphicByNameSorter(false);
    if (sorterElem != null) {
      sorter = readDatasetScanSorter(sorterElem);
    }

    // Read allProxies element (and addLatest element)
    Element addLatestElem = dsElem.getChild("addLatest", defNS);
    Element addProxiesElem = dsElem.getChild("addProxies", defNS);
    Map<String, ProxyDatasetHandler> allProxyDsHandlers;
    if (addLatestElem != null || addProxiesElem != null)
      allProxyDsHandlers = readDatasetScanAddProxies(addProxiesElem, addLatestElem, catalog);
    else
      allProxyDsHandlers = new HashMap<>();

    // Read addDatasetSize element.
    Element addDsSizeElem = dsElem.getChild("addDatasetSize", defNS);
    //boolean addDatasetSize = false; old way
    //if ( addDsSizeElem != null )
    //  addDatasetSize = true;
    boolean addDatasetSize = true;
    if (addDsSizeElem != null) {
      if (addDsSizeElem.getTextNormalize().equalsIgnoreCase("false"))
        addDatasetSize = false;
    }

    // Read addTimeCoverage element.
    List<DatasetEnhancer> childEnhancerList = new ArrayList<>();
    Element addTimeCovElem = dsElem.getChild("addTimeCoverage", defNS);
    if (addTimeCovElem != null) {
      DatasetEnhancer addTimeCovEnhancer = readDatasetScanAddTimeCoverage(addTimeCovElem);
      if (addTimeCovEnhancer != null)
        childEnhancerList.add(addTimeCovEnhancer);
    }

    // Read datasetEnhancerImpl elements (user defined implementations of DatasetEnhancer)
    List<Element> dsEnhancerElemList = dsElem.getChildren("datasetEnhancerImpl", defNS);
    for (Element elem : dsEnhancerElemList) {
      DatasetEnhancer o = readDatasetScanUserDefined(elem, DatasetEnhancer.class);
      if (o != null)
        childEnhancerList.add(o);
    }

    // Read catalogRefExpander element
//    Element catRefExpanderElem = dsElem.getChild( "catalogRefExpander", defNS );
    CatalogRefExpander catalogRefExpander = null;
//    if ( catRefExpanderElem != null )
//    {
//      catalogRefExpander = readDatasetScanCatRefExpander( catRefExpanderElem );
//    }

    InvDatasetScan datasetScan;
    try {
      datasetScan = new InvDatasetScan(parent, name, path, scanDir,
              configClassName, configObj,
              filter, identifier, namer,
              addDatasetSize, sorter, allProxyDsHandlers,
              childEnhancerList,
              catalogRefExpander);
      readDatasetInfo(catalog, datasetScan, dsElem, base);
      if (InvCatalogFactory.debugXML) System.out.println(" Dataset added: " + datasetScan.dump());

    } catch (Exception e) {
      logger.error("readDatasetScanNew(): failed to create DatasetScan", e);
      datasetScan = null;
    }

    return datasetScan;
  }

  CrawlableDatasetFilter readDatasetScanFilter(Element filterElem) {
    CrawlableDatasetFilter filter = null;  //lastModifiedLimit

    // Handle LastModifiedLimitFilter CrDsFilters.
    Attribute lastModLimitAtt = filterElem.getAttribute("lastModifiedLimit");
    if (lastModLimitAtt != null) {
      long lastModLimit;
      try {
        lastModLimit = lastModLimitAtt.getLongValue();
      } catch (DataConversionException e) {
        String tmpMsg = "readDatasetScanFilter(): bad lastModifedLimit value <" + lastModLimitAtt.getValue() + ">, couldn't parse into long: " + e.getMessage();
        factory.appendErr(tmpMsg);
        logger.warn(tmpMsg);
        return null;
      }
      return new LastModifiedLimitFilter(lastModLimit);
    }

    // Handle LogicalFilterComposer CrDsFilters.
    String compType = filterElem.getAttributeValue("logicalComp");
    if (compType != null) {
      List filters = filterElem.getChildren("filter", defNS);
      if (compType.equalsIgnoreCase("AND")) {
        if (filters.size() != 2) {
          String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for AND (2 expected).";
          factory.appendErr(tmpMsg);
          logger.warn(tmpMsg);
          return null;
        }
        filter = LogicalFilterComposer.getAndFilter(
                readDatasetScanFilter((Element) filters.get(0)),
                readDatasetScanFilter((Element) filters.get(1)));
      } else if (compType.equalsIgnoreCase("OR")) {
        if (filters.size() != 2) {
          String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for OR (2 expected).";
          factory.appendErr(tmpMsg);
          logger.warn(tmpMsg);
          return null;
        }
        filter = LogicalFilterComposer.getOrFilter(
                readDatasetScanFilter((Element) filters.get(0)),
                readDatasetScanFilter((Element) filters.get(1)));
      } else if (compType.equalsIgnoreCase("NOT")) {
        if (filters.size() != 1) {
          String tmpMsg = "readDatasetScanFilter(): wrong number of filters <" + filters.size() + "> for NOT (1 expected).";
          factory.appendErr(tmpMsg);
          logger.warn(tmpMsg);
          return null;
        }
        filter = LogicalFilterComposer.getNotFilter(
                readDatasetScanFilter((Element) filters.get(0)));
      }

      return filter;
    }

    // Handle user defined CrDsFilters.
    Element userDefElem = filterElem.getChild("crawlableDatasetFilterImpl", defNS);
    if (userDefElem != null) {
      filter = (CrawlableDatasetFilter) readDatasetScanUserDefined(userDefElem, CrawlableDatasetFilter.class);
    }

    // Handle MultiSelectorFilter and contained Selectors.
    else {
      List<MultiSelectorFilter.Selector> selectorList = new ArrayList<>();
      for (Element curElem : filterElem.getChildren()) {
        String regExpAttVal = curElem.getAttributeValue("regExp");
        String wildcardAttVal = curElem.getAttributeValue("wildcard");
        String lastModLimitAttVal = curElem.getAttributeValue("lastModLimitInMillis");
        if (regExpAttVal == null && wildcardAttVal == null && lastModLimitAttVal == null) {
          // If no regExp or wildcard attributes, skip this selector.
          logger.warn("readDatasetScanFilter(): no regExp, wildcard, or lastModLimitInMillis attribute in filter child <" + curElem.getName() + ">.");
        } else {
          // Determine if applies to atomic datasets, default true.
          boolean atomic = true;
          String atomicAttVal = curElem.getAttributeValue("atomic");
          if (atomicAttVal != null) {
            // If not "true", set to false.
            if (!atomicAttVal.equalsIgnoreCase("true"))
              atomic = false;
          }
          // Determine if applies to collection datasets, default false.
          boolean collection = false;
          String collectionAttVal = curElem.getAttributeValue("collection");
          if (collectionAttVal != null) {
            // If not "false", set to true.
            if (!collectionAttVal.equalsIgnoreCase("false"))
              collection = true;
          }

          // Determine if include or exclude selectors.
          boolean includer = true;
          if (curElem.getName().equals("exclude")) {
            includer = false;
          } else if (!curElem.getName().equals("include")) {
            logger.warn("readDatasetScanFilter(): unhandled filter child <" + curElem.getName() + ">.");
            continue;
          }

          // Determine if regExp or wildcard
          if (regExpAttVal != null) {
            selectorList.add(new MultiSelectorFilter.Selector(new RegExpMatchOnNameFilter(regExpAttVal), includer, atomic, collection));
          } else if (wildcardAttVal != null) {
            selectorList.add(new MultiSelectorFilter.Selector(new WildcardMatchOnNameFilter(wildcardAttVal), includer, atomic, collection));
          } else if (lastModLimitAttVal != null) {
            selectorList.add(new MultiSelectorFilter.Selector(new LastModifiedLimitFilter(Long.parseLong(lastModLimitAttVal)), includer, atomic, collection));
          }
        }
      }
      filter = new MultiSelectorFilter(selectorList);
    }

    return filter;
  }

  protected CrawlableDatasetLabeler readDatasetScanIdentifier(Element identifierElem) {
    CrawlableDatasetLabeler identifier;
    Element userDefElem = identifierElem.getChild("crawlableDatasetLabelerImpl", defNS);
    if (userDefElem != null) {
      identifier = (CrawlableDatasetLabeler) readDatasetScanUserDefined(userDefElem, CrawlableDatasetLabeler.class);
    } else {
      // Default is to add ID in standard way. Don't have alternates yet.
      return null;
    }

    return identifier;
  }

  protected CrawlableDatasetLabeler readDatasetScanNamer(Element namerElem) {
    CrawlableDatasetLabeler namer;
//    Element userDefElem = namerElem.getChild( "crawlableDatasetLabelerImpl", defNS );
//    if ( userDefElem != null )
//    {
//      namer = (CrawlableDatasetLabeler) readDatasetScanUserDefined( userDefElem, CrawlableDatasetLabeler.class );
//    }
//    else
//    {
    List<CrawlableDatasetLabeler> labelerList = new ArrayList<>();
    for (Element curElem : namerElem.getChildren()) {
      CrawlableDatasetLabeler curLabeler;

      String regExp = curElem.getAttributeValue("regExp");
      String replaceString = curElem.getAttributeValue("replaceString");
      if (curElem.getName().equals("regExpOnName")) {
        curLabeler = new RegExpAndReplaceOnNameLabeler(regExp, replaceString);
      } else if (curElem.getName().equals("regExpOnPath")) {
        curLabeler = new RegExpAndReplaceOnPathLabeler(regExp, replaceString);
      } else {
        logger.warn("readDatasetScanNamer(): unhandled namer child <" + curElem.getName() + ">.");
        continue;
      }
      labelerList.add(curLabeler);
    }
    namer = new MultiLabeler(labelerList);
//    }

    return namer;
  }

  protected CrawlableDatasetSorter readDatasetScanSorter(Element sorterElem) {
    CrawlableDatasetSorter sorter = null;
    Element userDefElem = sorterElem.getChild("crawlableDatasetSorterImpl", defNS);
    if (userDefElem != null) {
      sorter = (CrawlableDatasetSorter) readDatasetScanUserDefined(userDefElem, CrawlableDatasetSorter.class);
    } else {
      Element lexSortElem = sorterElem.getChild("lexigraphicByName", defNS);
      if (lexSortElem != null) {
        boolean increasing;
        String increasingString = lexSortElem.getAttributeValue("increasing");
        increasing = increasingString.equalsIgnoreCase("true");
        sorter = new LexigraphicByNameSorter(increasing);
      }
    }

    return sorter;
  }

  protected Map<String, ProxyDatasetHandler> readDatasetScanAddProxies(Element addProxiesElem, Element addLatestElem, InvCatalogImpl catalog) {
    Map<String, ProxyDatasetHandler> allProxyDsHandlers = new HashMap<>();

    // Handle old "addLatest" elements.
    if (addLatestElem != null) {
      // Check for simpleLatest element.
      Element simpleLatestElem = addLatestElem.getChild("simpleLatest", defNS);
      // Get a SimpleLatestDsHandler, use default values if element is null.
      ProxyDatasetHandler pdh = readDatasetScanAddLatest(simpleLatestElem, catalog);
      if (pdh != null)
        allProxyDsHandlers.put(pdh.getProxyDatasetName(), pdh);
    }

    // Handle all "addProxies" elements.
    if (addProxiesElem != null) {
      for (Element curChildElem : addProxiesElem.getChildren()) {
        ProxyDatasetHandler curPdh;

        // Handle "simpleLatest" child elements.
        if (curChildElem.getName().equals("simpleLatest")
                && curChildElem.getNamespace().equals(defNS)) {
          curPdh = readDatasetScanAddLatest(curChildElem, catalog);
        }

        // Handle "latestComplete" child elements.
        else if (curChildElem.getName().equals("latestComplete")
                && curChildElem.getNamespace().equals(defNS)) {
          // Get latest name.
          String latestName = curChildElem.getAttributeValue("name");
          if (latestName == null) {
            logger.warn("readDatasetScanAddProxies(): unnamed latestComplete, skipping.");
            continue;
          }

          // Does latest go on top or bottom of list.
          Attribute topAtt = curChildElem.getAttribute("top");
          boolean latestOnTop = true;
          if (topAtt != null) {
            try {
              latestOnTop = topAtt.getBooleanValue();
            } catch (DataConversionException e) {
              latestOnTop = true;
            }
          }

          // Get the latest service name.
          String serviceName = curChildElem.getAttributeValue("serviceName");
          if (serviceName == null) {
            logger.warn("readDatasetScanAddProxies(): no service name given in latestComplete.");
            continue;
          }
          InvService service = catalog.findService(serviceName);
          if (service == null) {
            logger.warn("readDatasetScanAddProxies(): named service <" + serviceName + "> not found.");
            continue;
          }

          // Get lastModifed limit.
          String lastModLimitVal = curChildElem.getAttributeValue("lastModifiedLimit");
          long lastModLimit;
          if (lastModLimitVal == null)
            lastModLimit = 60; // Default to one hour
          else
            lastModLimit = Long.parseLong(lastModLimitVal);

          // Get isResolver.
          String isResolverString = curChildElem.getAttributeValue("isResolver");
          boolean isResolver = true;
          if (isResolverString != null)
            if (isResolverString.equalsIgnoreCase("false"))
              isResolver = false;

          // Build the SimpleLatestProxyDsHandler and add to map.
          curPdh = new LatestCompleteProxyDsHandler(latestName, latestOnTop, service, isResolver, lastModLimit);
        } else {
          curPdh = null;
          // @todo Deal with allowing user defined inserters
          //Element userDefElem = addLatestElem.getChild( "proxyDatasetHandlerImpl", defNS );
        }

        // Add current proxy dataset handler to map if name is not already in map.
        if (curPdh != null) {
          if (allProxyDsHandlers.containsKey(curPdh.getProxyDatasetName())) {
            logger.warn("readDatasetScanAddProxies(): proxy map already contains key <" + curPdh.getProxyDatasetName() + ">, skipping.");
            continue;
          }
          allProxyDsHandlers.put(curPdh.getProxyDatasetName(), curPdh);
        }
      }
    }

    return allProxyDsHandlers;
  }

  /**
   * Return a SimpleLatestProxyDsHandler, use default values if element is null.
   *
   * @param simpleLatestElem the simpleLatest element
   * @param catalog          the catalog containing the simpleLatest element.
   * @return a SimpleLatestProxyDsHandler
   */
  private ProxyDatasetHandler readDatasetScanAddLatest(Element simpleLatestElem, InvCatalogImpl catalog) {
    // Default values is simpleLatestElem is null.
    ProxyDatasetHandler latestAdder = null;
    String latestName = "latest.xml";
    boolean latestOnTop = true;
    String latestServiceName = "latest";
    boolean isResolver = true;

    // If simpleLatestElem exists, read values.
    if (simpleLatestElem != null) {
      // Get latest name.
      String tmpLatestName = simpleLatestElem.getAttributeValue("name");
      if (tmpLatestName != null)
        latestName = tmpLatestName;

      // Does latest go on top or bottom of list.
      Attribute topAtt = simpleLatestElem.getAttribute("top");
      if (topAtt != null) {
        try {
          latestOnTop = topAtt.getBooleanValue();
        } catch (DataConversionException e) {
          latestOnTop = true;
        }
      }

      // Get the latest service name.
      String tmpLatestServiceName = simpleLatestElem.getAttributeValue("serviceName");
      if (tmpLatestServiceName != null)
        latestServiceName = tmpLatestServiceName;

      // Get isResolver.
      String isResolverString = simpleLatestElem.getAttributeValue("isResolver");
      if (isResolverString != null)
        if (isResolverString.equalsIgnoreCase("false"))
          isResolver = false;
    }

    // Build the SimpleLatestProxyDsHandler
    InvService service = catalog.findService(latestServiceName);
    if (service == null)
      logger.warn("readDatasetScanAddLatest(): named service <" + latestServiceName + "> not found.");
    else
      latestAdder = new SimpleLatestProxyDsHandler(latestName, latestOnTop, service, isResolver);

    return latestAdder;
  }

//  protected CatalogRefExpander readDatasetScanCatRefExpander( Element catRefExpanderElem )
//  {
//
//  }

  protected DatasetEnhancer readDatasetScanAddTimeCoverage(Element addTimeCovElem) {
    DatasetEnhancer timeCovEnhancer = null;

    String matchName = addTimeCovElem.getAttributeValue("datasetNameMatchPattern");
    String matchPath = addTimeCovElem.getAttributeValue("datasetPathMatchPattern");
    String subst = addTimeCovElem.getAttributeValue("startTimeSubstitutionPattern");
    String duration = addTimeCovElem.getAttributeValue("duration");
    if (matchName != null && subst != null && duration != null) {
      timeCovEnhancer = RegExpAndDurationTimeCoverageEnhancer
              .getInstanceToMatchOnDatasetName(matchName, subst, duration);
    } else if (matchPath != null && subst != null && duration != null) {
      timeCovEnhancer = RegExpAndDurationTimeCoverageEnhancer
              .getInstanceToMatchOnDatasetPath(matchPath, subst, duration);
    }

    return timeCovEnhancer;
  }

  private DatasetEnhancer readDatasetScanUserDefined(Element userDefElem, Class targetClass) {
    String className = userDefElem.getAttributeValue("className");
    Element configElem;
    List childrenElemList = userDefElem.getChildren();
    if (childrenElemList.size() == 1) {
      configElem = (Element) childrenElemList.get(0);
    } else if (childrenElemList.size() != 0) {
      logger.warn("readDatasetScanUserDefined(): config XML not a single element, using first element.");
      configElem = (Element) childrenElemList.get(0);
    } else {
      logger.debug("readDatasetScanUserDefined(): no config XML elements.");
      configElem = null;
    }

    try {
      // Get the Class instance for desired targetClass implementation.
      Class requestedClass = Class.forName(className);

      // Check that the requested Class is a target Class.
      if (!targetClass.isAssignableFrom(requestedClass)) {
        throw new IllegalArgumentException("Requested class <" + className + "> not an implementation of " + targetClass.getName() + ".");
      }

      // Instantiate the desired Object using that classes constructor with a
      // single Object argument.
      Class[] argTypes = {Object.class};
      Object[] args = {configElem};
      Constructor constructor = requestedClass.getConstructor(argTypes);

      return (DatasetEnhancer) constructor.newInstance(args);
    } catch (ClassNotFoundException e) {
      logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
      return null;
    } catch (NoSuchMethodException e) {
      logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
      return null;
    } catch (InstantiationException e) {
      logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
      return null;
    } catch (IllegalAccessException e) {
      logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
      return null;
    } catch (InvocationTargetException e) {
      logger.warn("readDatasetScanUserDefined(): exception creating user defined object <" + className + ">", e);
      return null;
    }
  }

  protected DataRootConfig readDatasetRoot(Element s) {
    String path = s.getAttributeValue("path");
    String dirLocation = s.getAttributeValue("location");
    if (dirLocation == null)
      dirLocation = s.getAttributeValue("dirLocation");
    dirLocation = expandAliasForPath(dirLocation);

    if (path != null) {
      if (path.charAt(0) == '/') path = path.substring(1);
      int last = path.length() - 1;
      if (path.charAt(last) == '/') path = path.substring(0, last);
    }

    if (dirLocation != null) {
      int last = dirLocation.length() - 1;
      if (dirLocation.charAt(last) != '/') dirLocation = dirLocation + '/';
    }

    return new DataRootConfig(path, dirLocation, s.getAttributeValue("cache"));
  }

  protected DateType readDate(Element elem) {
    if (elem == null) return null;
    String format = elem.getAttributeValue("format");
    String type = elem.getAttributeValue("type");
    return makeDateType(elem.getText(), format, type);
  }

  protected DateType makeDateType(String text, String format, String type) {
    if (text == null) return null;
    try {
      return new DateType(text, format, type);
    } catch (java.text.ParseException e) {
      factory.appendErr(" ** Parse error: Bad date format = " + text + "\n");
      return null;
    }
  }

  protected TimeDuration readDuration(Element elem) {
    if (elem == null) return null;
    String text = null;
    try {
      text = elem.getText();
      return new TimeDuration(text);
    } catch (java.text.ParseException e) {
      factory.appendErr(" ** Parse error: Bad duration format = " + text + "\n");
      return null;
    }
  }

  protected InvDocumentation readDocumentation(InvCatalog cat, Element s) {
    String href = s.getAttributeValue("href", xlinkNS);
    String title = s.getAttributeValue("title", xlinkNS);
    String type = s.getAttributeValue("type"); // not XLink type
    String content = s.getTextNormalize();

    URI uri = null;
    if (href != null) {
      try {
        uri = cat.resolveUri(href);
      } catch (Exception e) {
        factory.appendErr(" ** Invalid documentation href = " + href + " " + e.getMessage() + "\n");
      }
    }

    InvDocumentation doc = new InvDocumentation(href, uri, title, type, content);

    // LOOK XHTML ?? !!

    if (InvCatalogFactory.debugXML) System.out.println(" Documentation added: " + doc);
    return doc;
  }

  protected double readDouble(Element elem) {
    if (elem == null) return Double.NaN;
    String text = elem.getText();
    try {
      return Double.parseDouble(text);
    } catch (NumberFormatException e) {
      factory.appendErr(" ** Parse error: Bad double format = " + text + "\n");
      return Double.NaN;
    }
  }

  protected ThreddsMetadata.GeospatialCoverage readGeospatialCoverage(Element gcElem) {
    if (gcElem == null) return null;

    String zpositive = gcElem.getAttributeValue("zpositive");

    ThreddsMetadata.Range northsouth = readGeospatialRange(gcElem.getChild("northsouth", defNS), CDM.LAT_UNITS);
    ThreddsMetadata.Range eastwest = readGeospatialRange(gcElem.getChild("eastwest", defNS), CDM.LON_UNITS);
    ThreddsMetadata.Range updown = readGeospatialRange(gcElem.getChild("updown", defNS), "m");

    // look for names
    List<ThreddsMetadata.Vocab> names = new ArrayList<>();
    java.util.List<Element> list = gcElem.getChildren("name", defNS);
    for (Element e : list) {
      ThreddsMetadata.Vocab name = readControlledVocabulary(e);
      names.add(name);
    }

    return new ThreddsMetadata.GeospatialCoverage(eastwest, northsouth, updown, names, zpositive);
  }

  protected ThreddsMetadata.Range readGeospatialRange(Element spElem, String defUnits) {
    if (spElem == null) return null;

    double start = readDouble(spElem.getChild("start", defNS));
    double size = readDouble(spElem.getChild("size", defNS));
    double resolution = readDouble(spElem.getChild("resolution", defNS));

    String units = spElem.getChildText("units", defNS);
    if (units == null) units = defUnits;

    return new ThreddsMetadata.Range(start, size, resolution, units);
  }

  protected InvMetadata readMetadata(InvCatalog catalog, InvDatasetImpl dataset, Element mdataElement) {
    // there are 6 cases to deal with: threddsNamespace vs not & inline vs Xlink & hasConverter or not
    // (the hasConverter only applies when its not threddsNamespace, giving 6 cases)
    // this factory is the converter for threddsNamespace metadata
    //  and also handles non-threddsNamespace when there is no converter, in which case it just
    //   propagates the inline dom elements

    // figure out the namespace
    Namespace namespace;
    List inlineElements = mdataElement.getChildren();
    if (inlineElements.size() > 0) // look at the namespace of the children, if they exist
      namespace = ((Element) inlineElements.get(0)).getNamespace();
    else
      namespace = mdataElement.getNamespace(); // will be thredds

    String mtype = mdataElement.getAttributeValue("metadataType");
    String href = mdataElement.getAttributeValue("href", xlinkNS);
    String title = mdataElement.getAttributeValue("title", xlinkNS);
    String inheritedS = mdataElement.getAttributeValue("inherited");
    boolean inherited = (inheritedS != null) && inheritedS.equalsIgnoreCase("true");

    boolean isThreddsNamespace = ((mtype == null) || mtype.equalsIgnoreCase("THREDDS")) &&
            namespace.getURI().equals(XMLEntityResolver.CATALOG_NAMESPACE_10);

    // see if theres a converter for it.
    MetadataConverterIF metaConverter = factory.getMetadataConverter(namespace.getURI());
    if (metaConverter == null) metaConverter = factory.getMetadataConverter(mtype);
    if (metaConverter != null) {
      if (debugMetadataRead) System.out.println("found factory for metadata type = " + mtype + " namespace = " +
              namespace + "=" + metaConverter.getClass().getName());

      // see if theres any inline content
      Object contentObj;
      if (inlineElements.size() > 0) {
        contentObj = metaConverter.readMetadataContent(dataset, mdataElement);
        return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
                inherited, false, metaConverter, contentObj);

      } else { // otherwise it  must be an Xlink; defer reading
        return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
                namespace.getPrefix(), inherited, false, metaConverter);
      }
    }

    // the case where its not ThreddsMetadata, but theres no converter
    if (!isThreddsNamespace) {
      if (inlineElements.size() > 0) {
        // just hold onto the jdom elements as the "content" LOOK should be DOM?
        return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
                inherited, false, this, mdataElement);

      } else { // otherwise it must be an Xlink, never read
        return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
                namespace.getPrefix(), inherited, false, null);
      }

    }

    // the case where its ThreddsMetadata
    if (inlineElements.size() > 0) {
      ThreddsMetadata tmg = new ThreddsMetadata(false);
      readThreddsMetadata(catalog, dataset, mdataElement, tmg);
      return new InvMetadata(dataset, mtype, namespace.getURI(), namespace.getPrefix(),
              inherited, true, this, tmg);

    } else { // otherwise it  must be an Xlink; defer reading
      return new InvMetadata(dataset, href, title, mtype, namespace.getURI(),
              namespace.getPrefix(), inherited, true, this);
    }

  }

  /* MetadataConverterIF */
  public Object readMetadataContent(InvDataset dataset, org.jdom2.Element mdataElement) {
    InvMetadata m = readMetadata(dataset.getParentCatalog(), (InvDatasetImpl) dataset, mdataElement);
    return m.getThreddsMetadata();
  }

  private SAXBuilder saxBuilder;

  private Element readContentFromURL(java.net.URI uri) throws java.io.IOException {
    if (saxBuilder == null) saxBuilder = new SAXBuilder();
    Document doc;
    try {
      doc = saxBuilder.build(uri.toURL());
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    return doc.getRootElement();
  }

  // this is only called for ThredddsMetadata
  public Object readMetadataContentFromURL(InvDataset dataset, java.net.URI uri) throws java.io.IOException {
    Element elem = readContentFromURL(uri);
    Object contentObject = readMetadataContent(dataset, elem);
    if (debugMetadataRead) System.out.println(" convert to " + contentObject.getClass().getName());
    return contentObject;
  }

    /* open and read the referenced catalog XML
    if (debugMetadataRead) System.out.println(" readMetadataContentFromURL = " + url);
    org.w3c.dom.Element mdataElement = factory.readOtherXML( url);
    if (mdataElement == null) {
      factory.appendErr(" ** failed to read thredds metadata at = "+url+" for dataset"+dataset.getName()+"\n");
      return null;
    }

    Object contentObject = readMetadataContent( dataset, mdataElement);
    if (debugMetadataRead) System.out.println(" convert to " + contentObject.getClass().getName());
 return contentObject;  */

  // dummy LOOK
  public boolean validateMetadataContent(Object contentObject, StringBuilder out) {
    return true;
  }

  public void addMetadataContent(org.jdom2.Element mdataElement, Object contentObject) {
  }

  protected InvProperty readProperty(Element s) {
    String name = s.getAttributeValue("name");
    String value = s.getAttributeValue("value");
    return new InvProperty(name, value);
  }

  protected ThreddsMetadata.Source readSource(Element elem) {
    if (elem == null) return null;
    ThreddsMetadata.Vocab name = readControlledVocabulary(elem.getChild("name", defNS));
    Element contact = elem.getChild("contact", defNS);
    if (contact == null) {
      factory.appendErr(" ** Parse error: Missing contact element in = " + elem.getName() + "\n");
      return null;
    }
    return new ThreddsMetadata.Source(name, contact.getAttributeValue("url"), contact.getAttributeValue("email"));
  }

  protected InvService readService(Element s, URI baseURI) {
    String name = s.getAttributeValue("name");
    String type = s.getAttributeValue("serviceType");
    String serviceBase = s.getAttributeValue("base");
    String suffix = s.getAttributeValue("suffix");
    String desc = s.getAttributeValue("desc");

    InvService service = new InvService(name, type, serviceBase, suffix, desc);

    java.util.List<Element> propertyList = s.getChildren("property", defNS);
    for (Element e : propertyList) {
      InvProperty p = readProperty(e);
      service.addProperty(p);
    }

    java.util.List<Element> rootList = s.getChildren("datasetRoot", defNS);
    for (Element e : rootList) {
      InvProperty root = readDatasetRoot(e);
      service.addDatasetRoot(root);
    }

    // nested services
    java.util.List<Element> serviceList = s.getChildren("service", defNS);
    for (Element e : serviceList) {
      InvService ss = readService(e, baseURI);
      service.addService(ss);
    }

    if (InvCatalogFactory.debugXML) System.out.println(" Service added: " + service);
    return service;
  }

  protected double readDataSize(Element parent) {
    Element elem = parent.getChild("dataSize", defNS);
    if (elem == null) return Double.NaN;

    double size;
    String sizeS = elem.getText();
    try {
      size = Double.parseDouble(sizeS);
    } catch (NumberFormatException e) {
      factory.appendErr(" ** Parse error: Bad double format in size element = " + sizeS + "\n");
      return Double.NaN;
    }

    String units = elem.getAttributeValue("units");
    char c = Character.toUpperCase(units.charAt(0));
    if (c == 'K') size *= 1000;
    else if (c == 'M') size *= 1000 * 1000;
    else if (c == 'G') size *= 1000 * 1000 * 1000;
    else if (c == 'T') size *= 1000.0 * 1000 * 1000 * 1000;
    else if (c == 'P') size *= 1000.0 * 1000 * 1000 * 1000 * 1000;
    return size;
  }

  protected DateRange readTimeCoverage(Element tElem) {
    if (tElem == null) return null;

    DateType start = readDate(tElem.getChild("start", defNS));
    DateType end = readDate(tElem.getChild("end", defNS));
    TimeDuration duration = readDuration(tElem.getChild("duration", defNS));
    TimeDuration resolution = readDuration(tElem.getChild("resolution", defNS));

    try {
      return new DateRange(start, end, duration, resolution);
    } catch (java.lang.IllegalArgumentException e) {
      factory.appendWarning(" ** warning: TimeCoverage error = " + e.getMessage() + "\n");
      return null;
    }
  }

  protected void readThreddsMetadata(InvCatalog catalog, InvDatasetImpl dataset, Element parent, ThreddsMetadata tmg) {
    List<Element> list;

    // look for creators - kind of a Source
    list = parent.getChildren("creator", defNS);
    for (Element e : list) {
      tmg.addCreator(readSource(e));
    }

    // look for contributors
    list = parent.getChildren("contributor", defNS);
    for (Element e : list) {
      tmg.addContributor(readContributor(e));
    }

    // look for dates
    list = parent.getChildren("date", defNS);
    for (Element e : list) {
      DateType d = readDate(e);
      tmg.addDate(d);
    }

    // look for documentation
    list = parent.getChildren("documentation", defNS);
    for (Element e : list) {
      InvDocumentation doc = readDocumentation(catalog, e);
      tmg.addDocumentation(doc);
    }

    // look for keywords - kind of a controlled vocabulary
    list = parent.getChildren("keyword", defNS);
    for (Element e : list) {
      tmg.addKeyword(readControlledVocabulary(e));
    }

    // look for metadata
    java.util.List<Element> mList = parent.getChildren("metadata", defNS);
    for (Element e : mList) {
      InvMetadata m = readMetadata(catalog, dataset, e);
      if (m != null) {
        tmg.addMetadata(m);
      }
    }

    // look for projects - kind of a controlled vocabulary
    list = parent.getChildren("project", defNS);
    for (Element e : list) {
      tmg.addProject(readControlledVocabulary(e));
    }

    // look for properties
    list = parent.getChildren("property", defNS);
    for (Element e : list) {
      InvProperty p = readProperty(e);
      tmg.addProperty(p);
    }

    // look for publishers - kind of a Source
    list = parent.getChildren("publisher", defNS);
    for (Element e : list) {
      tmg.addPublisher(readSource(e));
    }

    // look for variables
    list = parent.getChildren("variables", defNS);
    for (Element e : list) {
      ThreddsMetadata.Variables vars = readVariables(catalog, dataset, e);
      tmg.addVariables(vars);
    }

    // can only be one each of these kinds
    ThreddsMetadata.GeospatialCoverage gc = readGeospatialCoverage(parent.getChild("geospatialCoverage", defNS));
    if (gc != null) tmg.setGeospatialCoverage(gc);

    DateRange tc = readTimeCoverage(parent.getChild("timeCoverage", defNS));
    if (tc != null) tmg.setTimeCoverage(tc);

    Element serviceNameElem = parent.getChild("serviceName", defNS);
    if (serviceNameElem != null) tmg.setServiceName(serviceNameElem.getText());

    Element authElem = parent.getChild("authority", defNS);
    if (authElem != null) tmg.setAuthority(authElem.getText());

    Element dataTypeElem = parent.getChild("dataType", defNS);
    if (dataTypeElem != null) {
      String dataTypeName = dataTypeElem.getText();
      if ((dataTypeName != null) && (dataTypeName.length() > 0)) {
        FeatureType dataType = FeatureType.getType(dataTypeName.toUpperCase());
        if (dataType == null) {
          factory.appendWarning(" ** warning: non-standard data type = " + dataTypeName + "\n");
        }
        tmg.setDataType(dataType);
      }
    }

    Element dataFormatElem = parent.getChild("dataFormat", defNS);
    if (dataFormatElem != null) {
      String dataFormatTypeName = dataFormatElem.getText();
      if ((dataFormatTypeName != null) && (dataFormatTypeName.length() > 0)) {
        DataFormatType dataFormatType = DataFormatType.findType(dataFormatTypeName);
        if (dataFormatType == null) {
          dataFormatType = DataFormatType.getType(dataFormatTypeName);
          factory.appendWarning(" ** warning: non-standard dataFormat type = " + dataFormatTypeName + "\n");
        }
        tmg.setDataFormatType(dataFormatType);
      }
    }

    double size = readDataSize(parent);
    if (!Double.isNaN(size))
      tmg.setDataSize(size);
  }

  protected ThreddsMetadata.Variable readVariable(Element varElem) {
    if (varElem == null) return null;

    String name = varElem.getAttributeValue("name");
    String desc = varElem.getText();
    String vocabulary_name = varElem.getAttributeValue("vocabulary_name");
    String units = varElem.getAttributeValue("units");
    String id = varElem.getAttributeValue("vocabulary_id");

    return new ThreddsMetadata.Variable(name, desc, vocabulary_name, units, id);
  }

  protected ThreddsMetadata.Variables readVariables(InvCatalog cat, InvDataset ds, Element varsElem) {
    if (varsElem == null) return null;

    String vocab = varsElem.getAttributeValue("vocabulary");
    String vocabHref = varsElem.getAttributeValue("href", xlinkNS);

    URI vocabUri = null;
    if (vocabHref != null) {
      try {
        vocabUri = cat.resolveUri(vocabHref);
      } catch (Exception e) {
        factory.appendErr(" ** Invalid Variables vocabulary URI = " + vocabHref + " " + e.getMessage() + "\n");
      }
    }

    java.util.List<Element> vlist = varsElem.getChildren("variable", defNS);

    String mapHref = null;
    URI mapUri = null;
    Element map = varsElem.getChild("variableMap", defNS);
    if (map != null) {
      mapHref = map.getAttributeValue("href", xlinkNS);
      try {
        mapUri = cat.resolveUri(mapHref);
      } catch (Exception e) {
        factory.appendErr(" ** Invalid Variables map URI = " + mapHref + " " + e.getMessage() + "\n");
      }
    }

    if ((mapUri != null) && vlist.size() > 0) { // cant do both
      factory.appendErr(" ** Catalog error: cant have variableMap and variable in same element (dataset = " +
              ds.getName() + "\n");
      mapUri = null;
    }

    ThreddsMetadata.Variables variables = new ThreddsMetadata.Variables(vocab, vocabHref, vocabUri, mapHref, mapUri);

    for (Element e : vlist) {
      ThreddsMetadata.Variable v = readVariable(e);
      variables.addVariable(v);
    }

    // read in variable map LOOK: would like to defer
    if (mapUri != null) {
      Element varsElement;
      try {
        varsElement = readContentFromURL(mapUri);
        List<Element> list = varsElement.getChildren("variable", defNS);
        for (Element e : list) {
          ThreddsMetadata.Variable v = readVariable(e);
          variables.addVariable(v);
        }
      } catch (IOException e) {
        logger.warn("Failure reading vaiable mapUri ", e);
      }

      /*org.w3c.dom.Element domElement = factory.readOtherXML(mapUri);
      if (domElement != null) {
        Element varsElement = toJDOM(domElement);
        List list = varsElement.getChildren("variable", defNS);
        for (int j = 0; j < list.size(); j++) {
          ThreddsMetadata.Variable v = readVariable( (Element) list.get(j));
          variables.addVariable(v);
        }
      } */

    }

    return variables;
  }

  /************************************************************************/
  // Writing XML from objects

  /**
   * Write the catalog as an XML document to the specified stream.
   *
   * @param catalog write this catalog
   * @param os      write to this OutputStream
   * @param raw     write raw file if true (for server configuration)
   * @throws IOException
   */
  public void writeXML(InvCatalogImpl catalog, OutputStream os, boolean raw) throws IOException {
    this.raw = raw;
    writeXML(catalog, os);
    this.raw = false;
  }

  private boolean raw = false;

  /**
   * Write the catalog as an XML document to the specified stream.
   *
   * @param catalog write this catalog
   * @param os      write to this OutputStream
   * @throws IOException
   */
  public void writeXML(InvCatalogImpl catalog, OutputStream os) throws IOException {
    // Output the document, use standard formatter
    //XMLOutputter fmt = new XMLOutputter();
    //fmt.setNewlines(true);
    //fmt.setIndent("  ");
    //fmt.setTrimAllWhite( true);
    XMLOutputter fmt = new XMLOutputter(org.jdom2.output.Format.getPrettyFormat());  // LOOK maybe compact ??
    fmt.output(writeCatalog(catalog), os);
  }

  public Document writeCatalog(InvCatalogImpl cat) {
    Element rootElem = new Element("catalog", defNS);
    Document doc = new Document(rootElem);

    // attributes
    if (cat.getName() != null)
      rootElem.setAttribute("name", cat.getName());
    rootElem.setAttribute("version", version);
    rootElem.addNamespaceDeclaration(xlinkNS);
    if (cat.getExpires() != null)
      rootElem.setAttribute("expires", cat.getExpires().toString());

    // services
    Iterator iter = cat.getServices().iterator();
    while (iter.hasNext()) {
      InvService service = (InvService) iter.next();
      rootElem.addContent(writeService(service));
    }

    // dataset roots
    if (raw) {
      iter = cat.getDatasetRoots().iterator();
      while (iter.hasNext()) {
        InvProperty p = (InvProperty) iter.next();
        rootElem.addContent(writeDatasetRoot(p));
      }
    }

    // properties
    iter = cat.getProperties().iterator();
    while (iter.hasNext()) {
      InvProperty p = (InvProperty) iter.next();
      rootElem.addContent(writeProperty(p));
    }

    // datasets
    iter = cat.getDatasets().iterator();
    while (iter.hasNext()) {
      InvDatasetImpl ds = (InvDatasetImpl) iter.next();
      if (ds instanceof InvDatasetScan)
        rootElem.addContent(writeDatasetScan((InvDatasetScan) ds));
      else if (ds instanceof InvCatalogRef)
        rootElem.addContent(writeCatalogRef((InvCatalogRef) ds));
      else
        rootElem.addContent(writeDataset(ds));
    }

    return doc;
  }

  private Element writeAccess(InvAccessImpl access) {
    Element accessElem = new Element("access", defNS);
    accessElem.setAttribute("urlPath", access.getUrlPath());
    if (access.getServiceName() != null)
      accessElem.setAttribute("serviceName", access.getServiceName());
    if (access.getDataFormatType() != null)
      accessElem.setAttribute("dataFormat", access.getDataFormatType().toString());

    if (access.hasDataSize())
      accessElem.addContent(writeDataSize(access.getDataSize()));

    return accessElem;
  }

  private Element writeCatalogRef(InvCatalogRef catRef) {
    Element catrefElem = new Element("catalogRef", defNS);
    catrefElem.setAttribute("href", catRef.getXlinkHref(), xlinkNS);
    String name = catRef.getName() == null ? "" : catRef.getName();
    catrefElem.setAttribute("title", name, xlinkNS);
    if (catRef.getID() != null)
      catrefElem.setAttribute("ID", catRef.getID());
    if (catRef.getRestrictAccess() != null)
      catrefElem.setAttribute("restrictAccess", catRef.getRestrictAccess());
    catrefElem.setAttribute("name", "");

    /* List list = catRef.getDocumentation();
    for (int j=0; j< list.size(); j++) {
      InvDocumentation doc = (InvDocumentation) list.get(j);
      catrefElem.addContent( writeDocumentation(doc, "documentation"));
    } */

    return catrefElem;
  }

  protected Element writeContributor(ThreddsMetadata.Contributor c) {
    Element elem = new Element("contributor", defNS);
    if (c.getRole() != null)
      elem.setAttribute("role", c.getRole());
    elem.setText(c.getName());
    return elem;
  }

  private Element writeControlledVocabulary(ThreddsMetadata.Vocab v, String name) {
    Element elem = new Element(name, defNS);
    if (v.getVocabulary() != null)
      elem.setAttribute("vocabulary", v.getVocabulary());
    elem.addContent(v.getText());
    return elem;
  }

  private Element writeDataset(InvDatasetImpl ds) {
    Element dsElem = new Element("dataset", defNS);

    if (ds instanceof InvDatasetImplProxy) {
      dsElem.setAttribute("name", ((InvDatasetImplProxy) ds).getAliasName());
      dsElem.setAttribute("alias", ds.getID());
      return dsElem;
    }

    writeDatasetInfo(ds, dsElem, true, raw);

    return dsElem;
  }

  private Element writeDatasetRoot(InvProperty prop) {
    Element drootElem = new Element("datasetRoot", defNS);
    drootElem.setAttribute("path", prop.getName());
    drootElem.setAttribute("location", prop.getValue());
    return drootElem;
  }

  private Element writeDatasetScan(InvDatasetScan ds) {
    Element dsElem;

    if (raw) {
      // Setup datasetScan element
      dsElem = new Element("datasetScan", defNS);
      writeDatasetInfo(ds, dsElem, false, true);
      dsElem.setAttribute("path", ds.getPath());
      dsElem.setAttribute("location", ds.getScanLocation());

      // Write datasetConfig element
      if (ds.getCrDsClassName() != null) {
        Element configElem = new Element("crawlableDatasetImpl", defNS);
        configElem.setAttribute("className", ds.getCrDsClassName());
        if (ds.getCrDsConfigObj() != null) {
          if (ds.getCrDsConfigObj() instanceof Element) {
            configElem.addContent((Element) ds.getCrDsConfigObj());
          }
        }
      }

      // Write filter element
      if (ds.getFilter() != null)
        dsElem.addContent(writeDatasetScanFilter(ds.getFilter()));

      // Write addID element
      //if ( ds.getIdentifier() != null )
      dsElem.addContent(writeDatasetScanIdentifier(ds.getIdentifier()));

      // Write namer element
      if (ds.getNamer() != null)
        dsElem.addContent(writeDatasetScanNamer(ds.getNamer()));

      // Write sort element
      if (ds.getSorter() != null)
        dsElem.addContent(writeDatasetScanSorter(ds.getSorter()));

      // Write addProxy element (and old addLatest element)
      if (!ds.getProxyDatasetHandlers().isEmpty())
        dsElem.addContent(writeDatasetScanAddProxies(ds.getProxyDatasetHandlers()));

      // Write addDatasetSize element
      if (ds.getAddDatasetSize())
        dsElem.addContent(new Element("addDatasetSize", defNS));

      // Write addTimeCoverage and datasetEnhancerImpl elements
      if (ds.getChildEnhancerList() != null)
        dsElem.addContent(writeDatasetScanEnhancer(ds.getChildEnhancerList()));

      // @todo Write catalogRefExpander elements
//      if ( ds.getCatalogRefExpander() != null )
//        dsElem.addContent( writeDatasetScanCatRefExpander( ds.getCatalogRefExpander()));
    } else {
      if (ds.isValid()) {
        dsElem = new Element("catalogRef", defNS);
        writeDatasetInfo(ds, dsElem, false, false);
        dsElem.setAttribute("href", ds.getXlinkHref(), xlinkNS);
        dsElem.setAttribute("title", ds.getName(), xlinkNS);
        dsElem.setAttribute("name", "");
        dsElem.addContent(writeProperty(new InvProperty("DatasetScan", "true")));
      } else {
        dsElem = new Element("dataset", defNS);
        dsElem.setAttribute("name", "** Misconfigured DatasetScan <" + ds.getPath() + "> **");
        dsElem.addContent(new Comment(ds.getInvalidMessage()));
      }
    }

    return dsElem;
  }

  Element writeDatasetScanFilter(CrawlableDatasetFilter filter) {
    Element filterElem = new Element("filter", defNS);
    if (filter.getClass().isAssignableFrom(MultiSelectorFilter.class) && filter.getConfigObject() != null) {
      for (Object o : ((List) filter.getConfigObject())) {
        MultiSelectorFilter.Selector curSelector = (MultiSelectorFilter.Selector) o;
        Element curSelectorElem;
        if (curSelector.isIncluder())
          curSelectorElem = new Element("include", defNS);
        else
          curSelectorElem = new Element("exclude", defNS);

        CrawlableDatasetFilter curFilter = curSelector.getFilter();
        if (curFilter instanceof WildcardMatchOnNameFilter) {
          curSelectorElem.setAttribute("wildcard", ((WildcardMatchOnNameFilter) curFilter).getWildcardString());
          curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
          curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
        } else if (curFilter instanceof RegExpMatchOnNameFilter) {
          curSelectorElem.setAttribute("regExp", ((RegExpMatchOnNameFilter) curFilter).getRegExpString());
          curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
          curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
        } else if (curFilter instanceof LastModifiedLimitFilter) {
          curSelectorElem.setAttribute("lastModLimitInMillis", Long.toString(((LastModifiedLimitFilter) curFilter).getLastModifiedLimitInMillis()));
          curSelectorElem.setAttribute("atomic", curSelector.isApplyToAtomicDataset() ? "true" : "false");
          curSelectorElem.setAttribute("collection", curSelector.isApplyToCollectionDataset() ? "true" : "false");
        } else
          curSelectorElem.addContent(new Comment("Unknown selector type <" + curSelector.getClass().getName() + ">."));

        filterElem.addContent(curSelectorElem);
      }
    } else {
      filterElem.addContent(writeDatasetScanUserDefined("crawlableDatasetFilterImpl", filter.getClass().getName(), filter.getConfigObject()));
    }

    return filterElem;
  }

  private Element writeDatasetScanNamer(CrawlableDatasetLabeler namer) {
    Element namerElem = null;
    if (namer != null) {
      namerElem = new Element("namer", defNS);
      if (namer instanceof MultiLabeler) {
        for (CrawlableDatasetLabeler curNamer : ((MultiLabeler) namer).getLabelerList()) {
          Element curNamerElem;
          if (curNamer instanceof RegExpAndReplaceOnNameLabeler) {
            curNamerElem = new Element("regExpOnName", defNS);
            curNamerElem.setAttribute("regExp", ((RegExpAndReplaceOnNameLabeler) curNamer).getRegExp());
            curNamerElem.setAttribute("replaceString", ((RegExpAndReplaceOnNameLabeler) curNamer).getReplaceString());
            namerElem.addContent(curNamerElem);
          } else if (curNamer instanceof RegExpAndReplaceOnPathLabeler) {
            curNamerElem = new Element("regExpOnPath", defNS);
            curNamerElem.setAttribute("regExp", ((RegExpAndReplaceOnPathLabeler) curNamer).getRegExp());
            curNamerElem.setAttribute("replaceString", ((RegExpAndReplaceOnPathLabeler) curNamer).getReplaceString());
            namerElem.addContent(curNamerElem);
          } else {
            String tmpMsg = "writeDatasetScanNamer(): unsupported namer <" + curNamer.getClass().getName() + ">.";
            logger.warn(tmpMsg);
            namerElem.addContent(new Comment(tmpMsg));
          }
        }
      } else {
        namerElem.addContent(writeDatasetScanUserDefined("crawlableDatasetLabelerImpl", namer.getClass().getName(), namer.getConfigObject()));
      }
    }

    return namerElem;
  }

  private Element writeDatasetScanIdentifier(CrawlableDatasetLabeler identifier) {
    Element identifierElem = new Element("addID", defNS);
    if (identifier != null) {
      if (identifier instanceof SimpleLatestProxyDsHandler) {
        return identifierElem;
      } else {
        identifierElem = new Element("addID", defNS);
        identifierElem.addContent(writeDatasetScanUserDefined("crawlableDatasetLabelerImpl", identifier.getClass().getName(), identifier.getConfigObject()));
      }
    }

    return identifierElem;
  }

  private Element writeDatasetScanAddProxies(Map<String, ProxyDatasetHandler> proxyDsHandlers) {
    Element addProxiesElem;

    // Write addLatest element if only proxyDsHandler and named "latest.xml".
    if (proxyDsHandlers.size() == 1 && proxyDsHandlers.containsKey("latest.xml")) {
      Object o = proxyDsHandlers.get("latest.xml");
      if (o instanceof SimpleLatestProxyDsHandler) {
        SimpleLatestProxyDsHandler pdh = (SimpleLatestProxyDsHandler) o;
        String name = pdh.getProxyDatasetName();
        boolean top = pdh.isLocateAtTopOrBottom();
        String serviceName = pdh.getProxyDatasetService(null).getName();

        addProxiesElem = new Element("addLatest", defNS);
        if (name.equals("latest.xml") && top && serviceName.equals("latest"))
          return addProxiesElem;
        else {
          Element simpleLatestElem = new Element("simpleLatest", defNS);

          simpleLatestElem.setAttribute("name", name);
          simpleLatestElem.setAttribute("top", top ? "true" : "false");
          simpleLatestElem.setAttribute("servicName", serviceName);
          addProxiesElem.addContent(simpleLatestElem);
          return addProxiesElem;
        }
      }
    }

    // Write "addProxies" element
    addProxiesElem = new Element("addProxies", defNS);
    for (Map.Entry<String, ProxyDatasetHandler> entry : proxyDsHandlers.entrySet()) {
      String curName = entry.getKey();
      ProxyDatasetHandler curPdh = entry.getValue();

      if (curPdh instanceof SimpleLatestProxyDsHandler) {
        SimpleLatestProxyDsHandler sPdh = (SimpleLatestProxyDsHandler) curPdh;

        Element simpleLatestElem = new Element("simpleLatest", defNS);

        simpleLatestElem.setAttribute("name", sPdh.getProxyDatasetName());
        simpleLatestElem.setAttribute("top", sPdh.isLocateAtTopOrBottom() ? "true" : "false");
        simpleLatestElem.setAttribute("servicName", sPdh.getProxyDatasetService(null).getName());
        addProxiesElem.addContent(simpleLatestElem);
      } else if (curPdh instanceof LatestCompleteProxyDsHandler) {
        LatestCompleteProxyDsHandler lcPdh = (LatestCompleteProxyDsHandler) curPdh;
        Element latestElem = new Element("latestComplete", defNS);
        latestElem.setAttribute("name", lcPdh.getProxyDatasetName());
        latestElem.setAttribute("top", lcPdh.isLocateAtTopOrBottom() ? "true" : "false");
        latestElem.setAttribute("servicName", lcPdh.getProxyDatasetService(null).getName());
        latestElem.setAttribute("lastModifiedLimit", Long.toString(lcPdh.getLastModifiedLimit()));
        addProxiesElem.addContent(latestElem);
      } else {
        logger.warn("writeDatasetScanAddProxies(): unknown type of ProxyDatasetHandler <" + curPdh.getProxyDatasetName() + ">.");
        // latestAdderElem.addContent( writeDatasetScanUserDefined( "datasetInserterImpl", latestAdder.getClass().getName(), latestAdder.getConfigObject() ) );
      }

    }
    return addProxiesElem;
  }

  private Element writeDatasetScanSorter(CrawlableDatasetSorter sorter) {
    Element sorterElem = new Element("sort", defNS);
    if (sorter instanceof LexigraphicByNameSorter) {
      Element lexElem = new Element("lexigraphicByName", defNS);
      lexElem.setAttribute("increasing", ((LexigraphicByNameSorter) sorter).isIncreasing() ? "true" : "false");
      sorterElem.addContent(lexElem);
    } else {
      sorterElem.addContent(writeDatasetScanUserDefined("crawlableDatasetSorterImpl", sorter.getClass().getName(), sorter.getConfigObject()));
    }

    return sorterElem;
  }

  private List<Element> writeDatasetScanEnhancer(List<DatasetEnhancer> enhancerList) {
    List<Element> enhancerElemList = new ArrayList<>();
    int timeCovCount = 0;
    for (DatasetEnhancer curEnhancer : enhancerList) {
      if (curEnhancer instanceof RegExpAndDurationTimeCoverageEnhancer) {
        if (timeCovCount > 0) {
          logger.warn("writeDatasetScanEnhancer(): More than one addTimeCoverage element, skipping.");
          continue;
        }
        timeCovCount++;
        Element timeCovElem = new Element("addTimeCoverage", defNS);
        RegExpAndDurationTimeCoverageEnhancer timeCovEnhancer = (RegExpAndDurationTimeCoverageEnhancer) curEnhancer;
        timeCovElem.setAttribute("datasetNameMatchPattern", timeCovEnhancer.getMatchPattern());
        timeCovElem.setAttribute("startTimeSubstitutionPattern", timeCovEnhancer.getSubstitutionPattern());
        timeCovElem.setAttribute("duration", timeCovEnhancer.getDuration());

        enhancerElemList.add(timeCovElem);
      } else {
        enhancerElemList.add(writeDatasetScanUserDefined("datasetEnhancerImpl", curEnhancer.getClass().getName(), curEnhancer.getConfigObject()));
      }
    }

    return enhancerElemList;
  }

  private Element writeDatasetScanUserDefined(String userDefName, String className, Object configObj) {
    Element userDefElem = new Element(userDefName, defNS);
    userDefElem.setAttribute("className", className);
    if (configObj != null) {
      if (configObj instanceof Element)
        userDefElem.addContent((Element) configObj);
      else
        userDefElem.addContent(new Comment("This class <" + className + "> not yet supported. This XML is missing configuration information (of type " + configObj.getClass().getName() + ")."));
    }

    return userDefElem;
  }

  private void writeDatasetInfo(InvDatasetImpl ds, Element dsElem, boolean doNestedDatasets, boolean showNcML) {
    dsElem.setAttribute("name", ds.getName());

    // other attributes, note the others get made into an element
    if ((ds.getCollectionType() != null) && (ds.getCollectionType() != CollectionType.NONE))
      dsElem.setAttribute("collectionType", ds.getCollectionType().toString());
    if (ds.isHarvest())
      dsElem.setAttribute("harvest", "true");
    if (ds.getID() != null)
      dsElem.setAttribute("ID", ds.getID());
    if (ds.getUrlPath() != null)
      dsElem.setAttribute("urlPath", ds.getUrlPath());
    if (ds.getRestrictAccess() != null)
      dsElem.setAttribute("restrictAccess", ds.getRestrictAccess());

    // services (local only)
    for (InvService service : ds.getServicesLocal()) {
      dsElem.addContent(writeService(service));
    }

    // thredds metadata
    writeThreddsMetadata(dsElem, ds.getLocalMetadata());
    writeInheritedMetadata(dsElem, ds.getLocalMetadataInheritable());
    // writeInheritedMetadata( dsElem, ds.getCat6Metadata()); // LOOK can we get rid of this?

    // access  (local only)
    for (InvAccess a : ds.getAccessLocal()) {
      dsElem.addContent(writeAccess( (InvAccessImpl) a));
    }

    if (showNcML && ds.getNcmlElement() != null) {
      org.jdom2.Element ncml = ds.getNcmlElement().clone();
      ncml.detach();
      dsElem.addContent(ncml);
    }

    if (!doNestedDatasets) return;

    // nested datasets
    for (InvDataset nested : ds.getDatasets()) {
      if (nested instanceof InvDatasetScan)
        dsElem.addContent(writeDatasetScan((InvDatasetScan) nested));
      else if (nested instanceof InvCatalogRef)
        dsElem.addContent(writeCatalogRef((InvCatalogRef) nested));
      else
        dsElem.addContent(writeDataset( (InvDatasetImpl) nested));
    }
  }

  protected Element writeDate(String name, DateType date) {
    Element dateElem = new Element(name, defNS);
    dateElem.addContent(date.getText());
    if (date.getType() != null)
      dateElem.setAttribute("type", date.getType());
    if (date.getFormat() != null)
      dateElem.setAttribute("format", date.getFormat());

    return dateElem;
  }

  private Element writeDocumentation(InvDocumentation doc, String name) {
    Element docElem = new Element(name, defNS);
    if (doc.getType() != null)
      docElem.setAttribute("type", doc.getType());

    if (doc.hasXlink()) {
      docElem.setAttribute("href", doc.getXlinkHref(), xlinkNS);
      if (!doc.getXlinkTitle().equals(doc.getURI().toString()))
        docElem.setAttribute("title", doc.getXlinkTitle(), xlinkNS);
    }

    String inline = doc.getInlineContent();
    if (inline != null)
      docElem.addContent(inline);
    return docElem;
  }

  public Element writeGeospatialCoverage(ThreddsMetadata.GeospatialCoverage gc) {
    Element elem = new Element("geospatialCoverage", defNS);
    if (gc.getZPositive().equals("down"))
      elem.setAttribute("zpositive", gc.getZPositive());

    if (gc.getNorthSouthRange() != null)
      writeGeospatialRange(elem, new Element("northsouth", defNS), gc.getNorthSouthRange());
    if (gc.getEastWestRange() != null)
      writeGeospatialRange(elem, new Element("eastwest", defNS), gc.getEastWestRange());
    if (gc.getUpDownRange() != null)
      writeGeospatialRange(elem, new Element("updown", defNS), gc.getUpDownRange());

    // serialize isGlobal
    java.util.List<ThreddsMetadata.Vocab> names = gc.getNames();
    ThreddsMetadata.Vocab global = new ThreddsMetadata.Vocab("global", null);
    if (gc.isGlobal() && !names.contains(global)) {
      names.add(global);
    } else if (!gc.isGlobal() && names.contains(global)) {
      names.remove(global);
    }

    for (ThreddsMetadata.Vocab name : names) {
      elem.addContent(writeControlledVocabulary(name, "name"));
    }

    return elem;
  }

  private void writeGeospatialRange(Element parent, Element elem, ThreddsMetadata.Range r) {
    if (r == null) return;

    elem.addContent(new Element("start", defNS).setText(Double.toString(r.getStart())));
    elem.addContent(new Element("size", defNS).setText(Double.toString(r.getSize())));
    if (r.hasResolution())
      elem.addContent(new Element("resolution", defNS).setText(Double.toString(r.getResolution())));
    if (r.getUnits() != null)
      elem.addContent(new Element("units", defNS).setText(r.getUnits()));

    parent.addContent(elem);
  }

  private Element writeMetadata(InvMetadata mdata) {
    Element mdataElem = new Element("metadata", defNS);
    if (mdata.getMetadataType() != null)
      mdataElem.setAttribute("metadataType", mdata.getMetadataType());
    if (mdata.isInherited())
      mdataElem.setAttribute("inherited", "true");

    String ns = mdata.getNamespaceURI();
    if ((ns != null) && !ns.equals(XMLEntityResolver.CATALOG_NAMESPACE_10)) {
      Namespace mdataNS = Namespace.getNamespace(mdata.getNamespacePrefix(), ns);
      mdataElem.addNamespaceDeclaration(mdataNS);
    }

    if (mdata.hasXlink()) {
      mdataElem.setAttribute("href", mdata.getXlinkHref(), xlinkNS);
      if (mdata.getXlinkTitle() != null)
        mdataElem.setAttribute("title", mdata.getXlinkTitle(), xlinkNS);

    } else if (mdata.getThreddsMetadata() != null) {
      writeThreddsMetadata(mdataElem, mdata.getThreddsMetadata());

    } else {
      // inline non-thredds case
      MetadataConverterIF converter = mdata.getConverter();
      if ((converter != null) && mdata.getContentObject() != null) {
        if (mdata.getContentObject() instanceof Element) { // special case
          Element mdataOrg = (Element) mdata.getContentObject();
          List<Element> children = mdataOrg.getChildren();
          for (Element child : children) {
            mdataElem.addContent( child.clone());
          }
        } else {
          //org.w3c.dom.Element dome = toDOM(mdataElem);
          converter.addMetadataContent(mdataElem, mdata.getContentObject());
          //mdataElem = toJDOM(dome);
          mdataElem.detach();
        }
      }
    }

    return mdataElem;
  }

  private Element writeProperty(InvProperty prop) {
    Element propElem = new Element("property", defNS);
    propElem.setAttribute("name", prop.getName());
    propElem.setAttribute("value", prop.getValue());
    return propElem;
  }

  protected Element writeSource(String elementName, ThreddsMetadata.Source p) {
    Element elem = new Element(elementName, defNS);

    elem.addContent(writeControlledVocabulary(p.getNameVocab(), "name"));

    Element contact = new Element("contact", defNS);
    if (p.getUrl() != null)
      contact.setAttribute("url", p.getUrl());
    if (p.getEmail() != null)
      contact.setAttribute("email", p.getEmail());
    elem.addContent(contact);

    return elem;
  }

  private Element writeService(InvService service) {
    Element serviceElem = new Element("service", defNS);
    serviceElem.setAttribute("name", service.getName());
    serviceElem.setAttribute("serviceType", service.getServiceType().toString());
    serviceElem.setAttribute("base", service.getBase());
    if ((service.getSuffix() != null) && (service.getSuffix().length() > 0))
      serviceElem.setAttribute("suffix", service.getSuffix());

    // properties
    for (InvProperty p : service.getProperties()) {
      serviceElem.addContent(writeProperty(p));
    }

    // services
    for (InvService nested : service.getServices()) {
      serviceElem.addContent(writeService(nested));
    }

    // dataset roots
    if (raw) {
      for (InvProperty p : service.getDatasetRoots()) {
        serviceElem.addContent(writeDatasetRoot(p));
      }
    }

    return serviceElem;
  }

  private Element writeDataSize(double size) {
    Element sizeElem = new Element("dataSize", defNS);

    // want exactly the number of bytes
    if (useBytesForDataSize) {
      sizeElem.setAttribute("units", "bytes");
      long bytes = (long) size;
      sizeElem.setText(Long.toString(bytes));
      return sizeElem;
    }

    // otherwise choose appropriate unit
    String unit;
    if (size > 1.0e15) {
      unit = "Pbytes";
      size *= 1.0e-15;
    } else if (size > 1.0e12) {
      unit = "Tbytes";
      size *= 1.0e-12;
    } else if (size > 1.0e9) {
      unit = "Gbytes";
      size *= 1.0e-9;
    } else if (size > 1.0e6) {
      unit = "Mbytes";
      size *= 1.0e-6;
    } else if (size > 1.0e3) {
      unit = "Kbytes";
      size *= 1.0e-3;
    } else {
      unit = "bytes";
    }

    sizeElem.setAttribute("units", unit);
    sizeElem.setText(ucar.unidata.util.Format.d(size, 4));

    return sizeElem;
  }

  /* protected void writeCat6InheritedMetadata( Element elem, ThreddsMetadata tmi) {
    if ((tmi.getDataType() == null) && (tmi.getServiceName() == null) &&
        (tmi.getAuthority() == null) && ( tmi.getProperties().size() == 0))
      return;

    Element mdataElem = new Element("metadata", defNS);
    mdataElem.setAttribute("inherited", "true");
    writeThreddsMetadata( mdataElem, tmi);
    elem.addContent( mdataElem);
  }  */

  protected void writeInheritedMetadata(Element elem, ThreddsMetadata tmi) {
    Element mdataElem = new Element("metadata", defNS);
    mdataElem.setAttribute("inherited", "true");
    writeThreddsMetadata(mdataElem, tmi);
    if (mdataElem.getChildren().size() > 0)
      elem.addContent(mdataElem);
  }

  protected void writeThreddsMetadata(Element elem, ThreddsMetadata tmg) {

    if (tmg.getServiceName() != null) {
      Element serviceNameElem = new Element("serviceName", defNS);
      serviceNameElem.setText(tmg.getServiceName());
      elem.addContent(serviceNameElem);
    }

    if (tmg.getAuthority() != null) {
      Element authElem = new Element("authority", defNS);
      authElem.setText(tmg.getAuthority());
      elem.addContent(authElem);
    }

    if ((tmg.getDataType() != null) && (tmg.getDataType() != FeatureType.ANY)) {
      Element dataTypeElem = new Element("dataType", defNS);
      dataTypeElem.setText(tmg.getDataType().toString());
      elem.addContent(dataTypeElem);
    }

    if ((tmg.getDataFormatType() != null) && (tmg.getDataFormatType() != DataFormatType.NONE)) {
      Element dataFormatElem = new Element("dataFormat", defNS);
      dataFormatElem.setText(tmg.getDataFormatType().toString());
      elem.addContent(dataFormatElem);
    }

    if (tmg.hasDataSize())
      elem.addContent(writeDataSize(tmg.getDataSize()));

    List<InvDocumentation> docList = tmg.getDocumentation();
    for (InvDocumentation doc : docList) {
      elem.addContent(writeDocumentation(doc, "documentation"));
    }

    List<ThreddsMetadata.Contributor> contribList = tmg.getContributors();
    for (ThreddsMetadata.Contributor c : contribList) {
      elem.addContent(writeContributor(c));
    }

    List<ThreddsMetadata.Source> creatorList = tmg.getCreators();
    for (ThreddsMetadata.Source p : creatorList) {
      elem.addContent(writeSource("creator", p));
    }

    List<ThreddsMetadata.Vocab> kewordList = tmg.getKeywords();
    for (ThreddsMetadata.Vocab v : kewordList) {
      elem.addContent(writeControlledVocabulary(v, "keyword"));
    }

    List<InvMetadata> mdList = tmg.getMetadata();
    for (InvMetadata m : mdList) {
      elem.addContent(writeMetadata(m));
    }

    List<ThreddsMetadata.Vocab> projList = tmg.getProjects();
    for (ThreddsMetadata.Vocab v : projList) {
      elem.addContent(writeControlledVocabulary(v, "project"));
    }

    List<InvProperty> propertyList = tmg.getProperties();
    for (InvProperty p : propertyList) {
      elem.addContent(writeProperty(p));
    }

    List<ThreddsMetadata.Source> pubList = tmg.getPublishers();
    for (ThreddsMetadata.Source p : pubList) {
      elem.addContent(writeSource("publisher", p));
    }

    List<DateType> dateList = tmg.getDates();
    for (DateType d : dateList) {
      elem.addContent(writeDate("date", d));
    }

    ThreddsMetadata.GeospatialCoverage gc = tmg.getGeospatialCoverage();
    if ((gc != null) && !gc.isEmpty())
      elem.addContent(writeGeospatialCoverage(gc));

    DateRange tc = tmg.getTimeCoverage();
    if (tc != null)
      elem.addContent(writeTimeCoverage(tc));

    List<ThreddsMetadata.Variables> varList = tmg.getVariables();
    for (ThreddsMetadata.Variables v : varList) {
      elem.addContent(writeVariables(v));
    }

    String varMapLink = tmg.getVariableMap();
    if (varMapLink != null) {
      Element velem = new Element("variableMap", defNS);
      velem.setAttribute("href", varMapLink, xlinkNS);
      velem.setAttribute("title", "variables", xlinkNS);
      elem.addContent(velem);
    }
  }

  protected Element writeTimeCoverage(DateRange t) {
    Element elem = new Element("timeCoverage", defNS);

    DateType start = t.getStart();
    DateType end = t.getEnd();
    TimeDuration duration = t.getDuration();
    TimeDuration resolution = t.getResolution();

    if (t.useStart() && (start != null) && !start.isBlank()) {
      Element startElem = new Element("start", defNS);
      startElem.setText(start.toString());
      elem.addContent(startElem);
    }

    if (t.useEnd() && (end != null) && !end.isBlank()) {
      Element telem = new Element("end", defNS);
      telem.setText(end.toString());
      elem.addContent(telem);
    }

    if (t.useDuration() && (duration != null) && !duration.isBlank()) {
      Element telem = new Element("duration", defNS);
      telem.setText(duration.toString());
      elem.addContent(telem);
    }

    if (t.useResolution() && (resolution != null) && !resolution.isBlank()) {
      Element telem = new Element("resolution", defNS);
      telem.setText(t.getResolution().toString());
      elem.addContent(telem);
    }

    return elem;
  }

  protected Element writeVariable(ThreddsMetadata.Variable v) {
    Element elem = new Element("variable", defNS);
    if (v.getName() != null)
      elem.setAttribute("name", v.getName());
    if (v.getDescription() != null) {
      String desc = v.getDescription().trim();
      if (desc.length() > 0)
        elem.setText(v.getDescription());
    }
    if (v.getVocabularyName() != null)
      elem.setAttribute("vocabulary_name", v.getVocabularyName());
    if (v.getUnits() != null)
      elem.setAttribute("units", v.getUnits());
    String id = v.getVocabularyId();
    if (id != null)
      elem.setAttribute("vocabulary_id", id);

    return elem;
  }

  protected Element writeVariables(ThreddsMetadata.Variables vs) {
    Element elem = new Element("variables", defNS);
    if (vs.getVocabulary() != null)
      elem.setAttribute("vocabulary", vs.getVocabulary());
    if (vs.getVocabHref() != null)
      elem.setAttribute("href", vs.getVocabHref(), xlinkNS);

    if (vs.getMapHref() != null) { // variable map
      Element mapElem = new Element("variableMap", defNS);
      mapElem.setAttribute("href", vs.getMapHref(), xlinkNS);
      elem.addContent(mapElem);

    } else { // inline variables
      List<ThreddsMetadata.Variable> varList = vs.getVariableList();
      for (ThreddsMetadata.Variable v : varList) {
        elem.addContent(writeVariable(v));
      }
    }

    return elem;
  }

  /* public org.w3c.dom.Element toDOM( Element elem) {
    try {
      if (domOut == null) domOut = new DOMOutputter();
      return domOut.output(elem);
    } catch (JDOMException e) {
      System.out.println("InvCatalogFactory6.readMetadata.toDom error " + e);
      return null;
    }
  }

  public Element toJDOM( org.w3c.dom.Element domElement) {
    return builder.build(domElement);
  }   */

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.mcidas;

import edu.wisc.ssec.mcidas.GridDirectory;
import edu.wisc.ssec.mcidas.McIDASException;
import edu.wisc.ssec.mcidas.McIDASUtil;

import ucar.nc2.iosp.grid.*;

import java.util.Formatter;

/**
 * A class to hold McIDAS grid record information
 *
 * @author dmurray
 */
public class McIDASGridRecord extends GridDirectory implements GridRecord {

  /**
   * offset to header
   */
  private int offsetToHeader;  // offset to header

  /**
   * grid definition
   */
  private McGridDefRecord gridDefRecord;

  /**
   * decimal scale
   */
  private int decimalScale = 0;

  /**
   * Create a grid header from the integer bits
   *
   * @param offset offset to grid header in file
   * @param header header words
   * @throws McIDASException Problem creating the record
   */
  public McIDASGridRecord(int offset, int[] header) throws McIDASException {
    super(header);
    gridDefRecord = new McGridDefRecord(header);
    offsetToHeader = offset;
  }

  /**
   * Get the first level of this GridRecord
   *
   * @return the first level value
   */
  public double getLevel1() {
    return getLevelValue();
  }

  /**
   * Get the second level of this GridRecord
   *
   * @return the second level value
   */
  public double getLevel2() {
    return getSecondLevelValue();
  }

  /**
   * Get the type for the first level of this GridRecord
   *
   * @return level type
   */
  public int getLevelType1() {
    // TODO:  flush this out
    int gribLevel = getDirBlock()[51];
    int levelType = 0;
    if (!((gribLevel == McIDASUtil.MCMISSING) || (gribLevel == 0))) {
      levelType = gribLevel;
    } else {
      levelType = 1;
    }
    return levelType;
  }

  /**
   * Get the type for the second level of this GridRecord
   *
   * @return level type
   */
  public int getLevelType2() {
    return getLevelType1();
  }

  /**
   * Get valid time offset (minutes) of this GridRecord
   *
   * @return time offset
   */
  public int getValidTimeOffset() {
    return getForecastHour();
  }

  /**
   * Get the parameter name
   *
   * @return parameter name
   */
  public String getParameterName() {
    return getParamName();
  }

  /**
   * Get the parameter description
   *
   * @return parameter description
   */
  public String getParameterDescription() {
    return getParamName();
  }

  /**
   * Get the decimal scale
   *
   * @return decimal scale
   */
  public int getDecimalScale() {
    return decimalScale;
  }

  /**
   * Get the grid def record id
   *
   * @return parameter name
   */
  public String getGridDefRecordId() {
    return gridDefRecord.toString();
  }

  /**
   * Get the grid def record id
   *
   * @return parameter name
   */
  public McGridDefRecord getGridDefRecord() {
    return gridDefRecord;
  }

  /**
   * Get the offset to the grid header (4 byte words)
   *
   * @return word offset
   */
  public int getOffsetToHeader() {
    return offsetToHeader;
  }

  /**
   * Had GRIB info?
   *
   * @return true if has grib info
   */
  public boolean hasGribInfo() {
    int gribSection = getDirBlock()[48];
    return (!((gribSection == McIDASUtil.MCMISSING) || (gribSection == 0)));
  }

  @Override
  public int getTimeUnit() {
    return 0;
  }

  @Override
  public String getTimeUdunitName() {
    return "minutes";
  }

  @Override
  // this determines how records get grouped into a cdm variable
  // unique parameter name and level type.
  public int cdmVariableHash() {
    return getParamName().hashCode() + 37 * getLevelType1();
  }

  @Override
  public String cdmVariableName(GridTableLookup lookup, boolean useLevel, boolean useStat) {
    Formatter f = new Formatter();
    f.format("%s", getParameterName());

    if (useLevel) {
      String levelName = lookup.getLevelName(this);
      if (levelName.length() != 0) {
        if (lookup.isLayer(this))
          f.format("_%s_layer", lookup.getLevelName(this));
        else
          f.format("_%s", lookup.getLevelName(this));
      }
    }

    return f.toString();
  }

}