/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib2;

import ucar.nc2.grib.GribNumbers;
import ucar.nc2.time.CalendarDate;
import ucar.unidata.io.RandomAccessFile;

import javax.annotation.concurrent.Immutable;
import java.io.IOException;

/**
 * The Identification section 1 for GRIB-2 files
 *
 * @author caron
 * @since 3/28/11
 */
@Immutable
public class Grib2SectionIdentification {
  private final int center_id;
  private final int subcenter_id;
  private final int master_table_version;
  private final int local_table_version;
  private final int significanceOfRT;
  private final int year, month, day, hour, minute, second;
  private final int productionStatus;
  private final int processedDataType;

  /**
   * Read Grib2SectionIndicator from raf.
   *
   * @param raf RandomAccessFile, with pointer at start of indicator section
   * @throws java.io.IOException on I/O error
   * @throws IllegalArgumentException if not a GRIB-2 record
   */
  public Grib2SectionIdentification(RandomAccessFile raf) throws IOException {
    long sectionEnd = raf.getFilePointer();

    // section 1 octet 1-4 (length of section)
    int length = GribNumbers.int4(raf);
    sectionEnd += length;

    int section = raf.read();
    if (section != 1)
      throw new IllegalArgumentException("Not a GRIB-2 Identification section");

    // Center  octet 6-7
    center_id = GribNumbers.int2(raf);

    // subCenter  octet 8-9
    subcenter_id = GribNumbers.int2(raf);

    // master table octet 10 (code table 1.0)
    master_table_version = raf.read();

    // local table octet 11  (code table 1.1)
    local_table_version = raf.read();

    // significanceOfRT octet 12 (code table 1.1)
    significanceOfRT = raf.read();

    // octets 13-19 (reference time of forecast)
    year = GribNumbers.int2(raf);
    month = raf.read();
    day = raf.read();
    hour = raf.read();
    minute = raf.read();
    second = raf.read();
    //refTime = CalendarDate.of(null, year, month, day, hour, minute, second);

    productionStatus = raf.read();
    processedDataType = raf.read();

    raf.seek(sectionEnd);
  }

  public Grib2SectionIdentification(int center_id, int subcenter_id, int master_table_version, int local_table_version,
                                    int significanceOfRT, int year, int month, int day, int hour, int minute, int second,
                                    int productionStatus, int processedDataType) {
    this.center_id = center_id;
    this.subcenter_id = subcenter_id;
    this.master_table_version = master_table_version;
    this.local_table_version = local_table_version;
    this.significanceOfRT = significanceOfRT;
    this.year = year;
    this.month = month;
    this.day = day;
    this.hour = hour;
    this.minute = minute;
    this.second = second;
    this.productionStatus = productionStatus;
    this.processedDataType = processedDataType;
  }

  /**
   * Identification of center (Common Code Table C-11)
   *
   * @return center id
   */
  public int getCenter_id() {
    return center_id;
  }

  /**
   * Identification of subcenter (allocated by center)
   *
   * @return subcenter
   */
  public int getSubcenter_id() {
    return subcenter_id;
  }

  /**
   * Parameter Table Version number (code table 1.0)
   *
   * @return master_table_version as int
   */
  public int getMaster_table_version() {
    return master_table_version;
  }

  /**
   * local table version number (code table 1.1)
   *
   * @return local_table_version as int
   */
  public int getLocal_table_version() {
    return local_table_version;
  }

  /**
   * Note 2 : if true, all entries come from the local table (!)
   * @return useLocalTablesOnly
   */
  public boolean useLocalTablesOnly() {
    return master_table_version == 255 && local_table_version != 0;
  }

  /**
   * Significance of Reference time (code table 1.2)
   *
   * @return significanceOfRT as int
   */
  public int getSignificanceOfRT() {
    return significanceOfRT;
  }

  /**
   * reference reference or base time as Dare.
   *
   * @return baseTime
   */
  public CalendarDate getReferenceDate() {
    return CalendarDate.of(null, year, month, day, hour, minute, second);
  }

  /**
   * production Status (code table 1.3)
   *
   * @return production Status
   */
  public int getProductionStatus() {
    return productionStatus;
  }

  /**
   * Get type Of Processed Data (code table 1.4)
   *
   * @return productType as int
   */
  public int getTypeOfProcessedData() {
    return processedDataType;
  }

  public int getYear() {
    return year;
  }

  public int getMonth() {
    return month;
  }

  public int getDay() {
    return day;
  }

  public int getHour() {
    return hour;
  }

  public int getMinute() {
    return minute;
  }

  public int getSecond() {
    return second;
  }

  @Override
  public String toString() {
    return "id {" +
            "center_id=" + center_id +
            ", subcenter_id=" + subcenter_id +
            ", master_table_version=" + master_table_version +
            ", local_table_version=" + local_table_version +
            '}';
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.catalog;

import javax.xml.parsers.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.*;
import org.jdom2.input.SAXBuilder;
import org.jdom2.Namespace;

import java.lang.reflect.*;
import java.io.*;
import java.util.*;

import ucar.nc2.constants.CDM;
import ucar.nc2.util.IO;

/**
 * Handles the interaction with JAXP, resolving dtd, schemas locally if possible.
 * Get a javax.xml.parsers.DocumentBuilder from here, allows you to validate or not.
 * <p>
 * The Crimson parser wont do schema validation.
 * NOTE: OLD AND CRUFTY
 * @author John Caron
 */

/*
  PROBLEM: in JAXP 1.2, setting JAXP_SCHEMA_LANGUAGE overrides the DTD
   even when theres no schema declared!

  Cant set JAXP_SCHEMA_LANGUAGE on catalog 0.6, get:
    ** Non-Fatal XML Error error(s) =
    *** XML parser error=cvc-elt.1: Cannot find the declaration of element 'catalog'.
  so validation is not done.

  Of course, you dont know what version until you start parsing !!!!

  "Set Schema Source" in JAXP 1.2: see http://java.sun.com/xml/jaxp/change-requests-11.html

  workaround: make a schema version of old DTD.
  problem: need Xerces parser (crimson wont do schema validation). So if you dont have, we dont even get DTD
    validation.
*/

/*
http://java.sun.com/xml/jaxp/reference/faqs/index.html

Q. How do I use a different JAXP compatible implementation?

The JAXP 1.1 API allows applications to plug in different JAXP compatible implementations of parsers or XSLT processors. For example, when an application wants to create a new JAXP DocumentBuilderFactory instance, it calls the staic method DocumentBuilderFactory.newInstance(). This causes a search for the name of a concrete subclass of DocumentBuilderFactory using the following order:

The value of a system property like javax.xml.parsers.DocumentBuilderFactory if it exists and is accessible.
The contents of the file $JAVA_HOME/jre/lib/jaxp.properties if it exists.
The Jar Service Provider discovery mechanism specified in the Jar File Specification. A jar file can have a resource (i.e. an embedded file) such as META-INF/services/javax.xml.parsers.DocumentBuilderFactory containing the name of the concrete class to instantiate.
The fallback platform default implementation.
Of the above ways to specify an implementation, perhaps the most useful is the jar service provider mechanism. To use this mechanism, place the implementation jar file on your classpath. For example, to use Xerces 1.4.4 instead of the version of Crimson which is bundled with JDK 1.4 (Java Development Kit version 1.4), place xerces.jar in your classpath. This mechanism also works with older versions of the JDK which do not bundle JAXP. If you are using JDK 1.4 and above, see this question for potential problems.

*/

/*
 VERSION notes

  Havent been able to detect how to know what version parser you have. May be new feature in JAXP 1.3

  Java 1.4.2 uses JAXP 1.1, which doesnt do W3S Schema validation.
              (see http://xml.apache.org/~edwingo/jaxp-ri-1.2.0-fcs/docs/)
    when you call factory.setAttribute() get message "No attributes are implemented"
    when you validate=true, get warning "Valid documents must have a <!DOCTYPE declaration.", plus lots of errors,
    but no fatal errors.

  It appears that "Tiger should support the latest version of JAXP...JAXP will be update to conform to the
    new specifications: XML 1.1 Namespaces 1.1 SAX 2.0.1". Note they dont actually say what that is. My
    JavaOne notes claim JAXP 1.3 (JSR 206). Hard to tell what they've added: access to version info and
    perhaps XInclude.

  JAXP 1.2 available in WSDP 1.3.
  Install JAXP 1.2 into J2SE 1.4, see http://java.sun.com/webservices/docs/1.2/jaxp/Updating.html

  "Version 1.4 of the Java 2 platform has a JAXP 1.1 implementation built in. (JAXP 1.1 has a smaller
   application footprint, but JAXP 1.2 implements XML Schema and the transform compiler, XSLTC.)
   Because the built-in libraries are accessed before any classpath entries, you can only access the
   updated libraries by using the Java platform's Endorsed Standards mechanism.
  There are two ways to use the endorsed standards mechanism. The first way is to copy all of the jar
    files except jaxp-api.jar into <JAVA_HOME>/jre/lib/endorsed/
  Note:
   The jaxp-api.jar file should not be copied, because it contains high-level factory APIs that are not
    subject to change.
  Alternatively, you can use the java.endorsed.dirs system property to dynamically add those jar files to
    the JVM when you start your program. Using that system property gives you flexibility of using different
    implementations for different applications.  You specify that property using
     -Djava.endorsed.dirs=yourDirectoryPath.
  Finally, although the endorsed directory mechanism guarantees that the specified jars will be searched
   before the internal classes in the VM (which are searched before the classpath), the order in which the
   jar files are searched is indeterminate. For that reason, there should be no overlaps in the classes
   specified using that property. For more information, see the Endorsed Standards documentation for the
   1.4 version of the Java platform."

  Webstart see http://forum.java.sun.com/thread.jsp?forum=38&thread=231995

"With my workaround for the context classloader in-place (see bug report) all I've ever needed to switch
 between xerces and crimson is the single property;
 <property name="javax.xml.parsers.SAXParserFactory" value="org.apache.xerces.jaxp.SAXParserFactoryImpl"/>

added into my jnlp; along with the jars, jaxp.jar, xercesImpl.jar & xmlParserAPIs.jar. (jnlp also needs to
   work under 1.2 & 1.3 - and it does!).
Hardcoding the parser just defeats the purpose of JAXP, although I too am wondering just which parser is
  going to be the default in the future - I was expecting xerces in 1.4.1 as it's the only way to handle
  schemas, maybe when JAXP gets updated with its schema properties/features I guess. "

----------

Add these to the jnlp file: (see Viewer.jnlp):

 <property name="javax.xml.parsers.SAXParserFactory" value="org.apache.xerces.jaxp.SAXParserFactoryImpl"/>
 <property name="javax.xml.parsers.DocumentBuilderFactory" value="org.apache.xerces.jaxp.DocumentBuilderFactoryImpl"/>

 these 2 classes are in xercesImpl.jar

  org.apache.xerces.* not in 1.4.2/rt.jar
*/

/* 4/26/04
   not apparently picking up the unique/key/keyref constraints in schema.
   if catalog has 2 datasets with same ID; dont get an error.
   does give an error using catalog 0.6, which still uses schema, but uses xs:ID type.
 */

/* 5/10/04
  tomcat (5.0.19, 5.0.24) ships with "Xerces-J 2.6.1" in common/endorsed
*/

public class XMLEntityResolver implements org.xml.sax.EntityResolver {
  static private final Logger logger = LoggerFactory.getLogger(XMLEntityResolver.class);

  //static private boolean debugEntityResolution = false; //, debugFactory = false, debugMessages = false;
  static private Map<String, String> entityHash = new HashMap<>();

  // schema validation
  //static private boolean schemaValidationOk = true;
  //static private final String JAXP_SCHEMA_LANGUAGE = "http://java.sun.com/xml/jaxp/properties/schemaLanguage";
  //static public final String W3C_XML_NAMESPACE = "http://www.w3.org/2001/XMLSchema";
 // static private final String JAXP_SCHEMA_SOURCE = "http://java.sun.com/xml/jaxp/properties/schemaSource";

  // catalog namespaces
  //static public final String CATALOG_NAMESPACE_06 = "http://www.unidata.ucar.edu/thredds";
  static public final String CATALOG_NAMESPACE_10 = "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0";

  // dqc namespaces (still referenced in legacy)
  //static public final String DQC_NAMESPACE_02 = "http://www.unidata.ucar.edu/schemas/thredds/queryCapability";
  //static public final String DQC_NAMESPACE_03 = "http://www.unidata.ucar.edu/namespaces/thredds/queryCapability/v0.3";
  static public final String DQC_NAMESPACE_04 = "http://www.unidata.ucar.edu/namespaces/thredds/queryCapability/v0.4";

  // nj22 namespaces
  static public final String NJ22_NAMESPACE = "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";
  static public final Namespace ncmlNS = Namespace.getNamespace("xlink", NJ22_NAMESPACE);

  // other namespaces
  static public final String XLINK_NAMESPACE = "http://www.w3.org/1999/xlink";
  static public final Namespace xlinkNS = Namespace.getNamespace("xlink", XLINK_NAMESPACE);

  static public final Namespace xsiNS = Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");

  static {
    /* Get Document Builder Factory
    docBuilderFactory = DocumentBuilderFactory.newInstance();
    docBuilderFactory.setNamespaceAware(true);

    // try to get schema validation
    if (schemaValidationOk) {
      try {
        docBuilderFactory.setAttribute(JAXP_SCHEMA_LANGUAGE, W3C_XML_NAMESPACE);
        docBuilderFactory.setAttribute(JAXP_SCHEMA_SOURCE,
           new String[] { CATALOG_NAMESPACE_06, CATALOG_NAMESPACE_10, XLINK_NAMESPACE, DQC_NAMESPACE_02, DQC_NAMESPACE_03 } );
      }
      catch (IllegalArgumentException e) {
        System.out.println("***Warning: XML Parser does not support JAXP 1.2. Library will continue without validation.\n"+e);
        schemaValidationOk = false;
      }
    }

    if (debugFactory) showFactoryInfo( docBuilderFactory); */

    // set up entity resolution - see MyEntityResolver class
    // catalog 0.6 dtd
    /* initEntity( "http://www.unidata.ucar.edu/projects/THREDDS/xml/InvCatalog.0.6.dtd",
                "/resources/thredds/schemas/InvCatalog.0.6.dtd",
                "http://www.unidata.ucar.edu/projects/THREDDS/xml/InvCatalog.0.6.dtd");  */

    // catalog 1.0 schema
    initEntity( CATALOG_NAMESPACE_10,
                "/resources/thredds/schemas/InvCatalog.1.2.xsd",
                "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.1.2.xsd");

    /* catalog 0.6 schema
    initEntity( CATALOG_NAMESPACE_06,
                "/resources/thredds/schemas/InvCatalog.0.6.xsd",
                "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.0.6.xsd");

    // DQC schema 0.2
    initEntity( DQC_NAMESPACE_02,
                "/resources/thredds/schemas/queryCapability.0.2.xsd",
                "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.2.xsd");

    // DQC schema 0.3
    initEntity( DQC_NAMESPACE_03,
                "/resources/thredds/schemas/queryCapability.0.3.xsd",
                "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.3.xsd"); */

    // DQC schema 0.4
    initEntity( DQC_NAMESPACE_04,
                "/resources/thredds/schemas/queryCapability.0.4.xsd",
                "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.4.xsd");

    // nj22 schema
    initEntity( NJ22_NAMESPACE,
                "/resources/nj22/schemas/ncml-2.2.xsd",
                "http://www.unidata.ucar.edu/schemas/netcdf/ncml-2.2.xsd");

    // XLink schema
    initEntity( XLINK_NAMESPACE,
                "/resources/thredds/schemas/xlink.xsd",
                "http://www.unidata.ucar.edu/schemas/other/xlink.xsd");

    /* catgen 0.5 dtd
    initEntity( "http://www.unidata.ucar.edu/projects/THREDDS/xml/CatalogGenConfig.0.5.dtd",
                "/resources/thredds/schemas/CatalogGenConfig.0.5.dtd",
                "http://www.unidata.ucar.edu/projects/THREDDS/xml/CatalogGenConfig.0.5.dtd"); */

  }

  static private boolean hasXerces = true;

  static private String externalSchemas;
  static public String getExternalSchemas() {
    if (externalSchemas == null) {
      externalSchemas =
        XMLEntityResolver.CATALOG_NAMESPACE_10 + " " + "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.1.0.7.xsd" +" "
        + XMLEntityResolver.NJ22_NAMESPACE + " " + "http://www.unidata.ucar.edu/schemas/netcdf/ncml-2.2.xsd" +" "
        + XMLEntityResolver.DQC_NAMESPACE_04 + " " + "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.4.xsd" +" "
        /* + XMLEntityResolver.DQC_NAMESPACE_03 + " " + "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.3.xsd" +" " +
        XMLEntityResolver.DQC_NAMESPACE_02 + " " + "http://www.unidata.ucar.edu/schemas/thredds/queryCapability.0.2.xsd" +" " +
        XMLEntityResolver.CATALOG_NAMESPACE_06 + " " + "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.0.6.xsd" */;
    }
    return externalSchemas;
  }

  /**
   * Add an entity for resolution. Specify a local resource, and/or a URL. Look
   * for the local Resource first.
   *
   * @param entityName name of entity, eg the namespace String
   * @param resourceName resolve using this Resource, found on the class path
   * @param urlName resolve using this Resource, found on the class path
   */
  static public void initEntity( String entityName, String resourceName, String urlName) {
    String entity = null;

    try (InputStream is = ucar.nc2.util.IO.getFileResource( resourceName)) { // try to read from local file resource, eg from catalog.jar
      ByteArrayOutputStream sbuff = new ByteArrayOutputStream(3000);
      if (is != null) {
        IO.copy(is, sbuff);
        entity = new String(sbuff.toByteArray(), CDM.utf8Charset);
        logger.debug(" *** entity " + entityName + " mapped to local resource at " + resourceName);

      } else if (urlName != null) { // otherwise, get from network
        entity = IO.readURLcontentsWithException(urlName);
        logger.debug(" *** entity "+entityName+" mapped to remote URL at "+urlName);
      }

    } catch (IOException e) {
      System.out.println(" *** FAILED to map entity "+entityName+" locally at "+resourceName+" or remotely at "+urlName);
      // e.printStackTrace();
    }

    entityHash.put( entityName, entity);
    entityHash.put( urlName, entity); // also map it to the remote URL
  }

  static public String getDocumentBuilderFactoryVersion() {
    try {
      Class version = Class.forName("org.apache.xerces.impl.Version");
      Method m = version.getMethod("getVersion", (Class []) null);
      return (String) m.invoke(null, (Object []) null);
    } catch (Exception e) {
      return "Error= "+e.getMessage();
    }
  }

  static private void showFactoryInfo(DocumentBuilderFactory factory) {
    System.out.println("------------------------");
    System.out.println("DocumentBuilderFactory class= "+factory.getClass().getName());
    try {
      //ClassLoader cl = factory.getClass().getClassLoader();
      //Class version = cl.loadClass("org.apache.xerces.impl.Version");
      Class version = Class.forName("org.apache.xerces.impl.Version");
      Method m = version.getMethod("getVersion", (Class []) null);
      System.out.println(" org.apache.xerces.impl.Version.version()="+m.invoke(null, (Object []) null));

      //thredds.util.reflect.ProxyGenerator.showMethods( factory.getClass(), System.out);
    } catch (Exception e) {
      e.printStackTrace();
    }
    System.out.println();

    System.out.println(" factory.isValidating()="+factory.isValidating());
    System.out.println(" factory.isNamespaceAware()="+factory.isNamespaceAware());
    System.out.println(" factory.isIgnoringElementContentWhitespace()="+factory.isIgnoringElementContentWhitespace());
    System.out.println(" factory.isExpandEntityReferences()="+factory.isExpandEntityReferences());
    System.out.println(" factory.isIgnoringComments()="+factory.isIgnoringComments());
    System.out.println(" factory.isCoalescing()="+factory.isCoalescing());

  }

  static private void showBuilderInfo( DocumentBuilder builder) {
    System.out.println("-----------------------");
    System.out.println(" builder.isValidating()="+builder.isValidating());
    System.out.println(" builder.isNamespaceAware()="+builder.isNamespaceAware());

    System.out.println("DocumentBuilder class= "+builder.getClass().getName());
    /* try {
      thredds.util.reflect.ProxyGenerator.showMethods( builder.getClass(), System.out);
      thredds.util.reflect.ProxyGenerator.showMethods( builder.getDOMImplementation().getClass(), System.out);
    } catch (Exception e) {
      e.printStackTrace();
    } */

  }

  ////////////////////////////////////////////////////////////////////////////////////////////////
  /* private DocumentBuilder builder; // JAXP parser

  private StringBuffer warnMessages = new StringBuffer();
  private StringBuffer errMessages = new StringBuffer();
  private StringBuffer fatalMessages = new StringBuffer();
  private String version = ""; */

  /* Constructor.
   * @param validate : do XML validation or not.
   *
  public XMLEntityResolver(boolean validate) {

    try {
      synchronized (docBuilderFactory) { // Only use one DocumentBuilderFactory, so needs to be synchronized
          docBuilderFactory.setValidating(validate && schemaValidationOk);
          builder = docBuilderFactory.newDocumentBuilder();
      }
      builder.setEntityResolver(this);
      builder.setErrorHandler(new MyErrorHandler());

      if (debugFactory) showBuilderInfo(builder);

    } catch (ParserConfigurationException e) {
      System.out.println("The underlying parser does not support the requested features.");
      throw new UnsupportedOperationException();

    } catch (FactoryConfigurationError e) {
      System.out.println("Error occurred obtaining Document Builder Factory.");
      throw new UnsupportedOperationException();
    }

  } */

  private SAXBuilder saxBuilder;
  private StringBuilder warnMessages = new StringBuilder();
  private StringBuilder errMessages = new StringBuilder();
  private StringBuilder fatalMessages = new StringBuilder();

  public XMLEntityResolver(boolean validate) {
    saxBuilder = hasXerces ? new SAXBuilder( validate) : new SAXBuilder("org.apache.xerces.parsers.SAXParser", validate);
    saxBuilder.setErrorHandler( new MyErrorHandler() );
    if (validate) {
      saxBuilder.setFeature( "http://apache.org/xml/features/validation/schema", true);
      saxBuilder.setProperty("http://apache.org/xml/properties/schema/external-schemaLocation",
        XMLEntityResolver.getExternalSchemas());
    }
    saxBuilder.setEntityResolver( this);
  }

  public SAXBuilder getSAXBuilder() { return saxBuilder; }
  public StringBuilder getWarningMessages() { return warnMessages; }
  public StringBuilder getErrorMessages() { return errMessages; }
  public StringBuilder getFatalMessages() { return fatalMessages; }
  // public String getVersion() { return version; }

  // we read the DTD/schema locally if we can.
  public org.xml.sax.InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
    logger.debug("  publicId="+publicId+" systemId="+systemId);
    String entity = entityHash.get( systemId);
    if (entity != null) {
      logger.debug(" *** resolved  with local copy");
      return new MyInputSource(entity);
    }

    if (systemId.contains("InvCatalog.0.6.dtd")) {
      entity = entityHash.get( "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.0.6.dtd");
      if (entity != null) {
        logger.debug(" *** resolved2 with local copy");
        return new MyInputSource(entity);
      }
    }

    logger.debug(" *** not resolved");
    return null;
  }

  private class MyErrorHandler implements org.xml.sax.ErrorHandler {

    public void warning(SAXParseException e) throws SAXException {
      warnMessages.append("*** XML parser warning ").append(showError(e)).append("\n");
    }

    public void error(SAXParseException e) throws SAXException {
      errMessages.append("*** XML parser error ").append(showError(e)).append("\n");
    }

    public void fatalError(SAXParseException e) throws SAXException {
      fatalMessages.append("*** XML parser fatalError ").append(showError(e)).append("\n");
    }

    private String showError( SAXParseException e) {
      return "("+e.getLineNumber()+":"+e.getColumnNumber()+")= "+e.getMessage();
    }

  }

  private static class MyInputSource extends org.xml.sax.InputSource {
    MyInputSource( String entity) {
      setCharacterStream(new StringReader(entity));
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.stream;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import ucar.ma2.Array;
import ucar.ma2.ArrayStructureBB;
import ucar.ma2.ArrayStructureBBsection;
import ucar.ma2.DataType;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Section;
import ucar.ma2.StructureMembers;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Structure;
import ucar.nc2.Variable;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.IO;
import ucar.unidata.io.RandomAccessFile;

/**
 * IOSP to read an ncStream that has been written to a file.
 * Proof of concept for making ncStream-encoded file into a version of the netcdf format.
 */
public class NcStreamIosp extends AbstractIOServiceProvider {
  private static final boolean debug = false;

  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    raf.seek(0);
    if (!readAndTest(raf, NcStream.MAGIC_START)) return false; // must start with these 4 bytes
    byte[] b = new byte[4];
    raf.readFully(b);
    return test(b, NcStream.MAGIC_HEADER) || test(b, NcStream.MAGIC_DATA); // immed followed by one of these
  }

  public String getFileTypeId() {
    return "ncstream";
  }

  public String getFileTypeDescription() {
    return "netCDF streaming protocol";
  }

  public int getVersion() {
    return version;
  }

  //////////////////////////////////////////////////////////////////////
  private int version;

  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    super.open(raf, ncfile, cancelTask);
    openDebug(raf, ncfile, null);
  }

  private static class DataStorage {
    int size;
    long filePos;
    Section section;
    boolean isVlen, isDeflate;
    ByteOrder bo;
    int nelems, uncompressedLen;
    NcStreamProto.StructureData sdata;

    DataStorage(int size, long filePos, NcStreamProto.Data dproto) {
      this.size = size;
      this.filePos = filePos;
      section = NcStream.decodeSection(dproto.getSection());
      nelems = (int) section.computeSize();
      bo = NcStream.decodeDataByteOrder(dproto);
      isVlen = dproto.getVdata();
      isDeflate = dproto.getCompress() == NcStreamProto.Compress.DEFLATE;
      if (isDeflate)
        uncompressedLen = dproto.getUncompressedSize();
    }

    @Override
    public String toString() {
      return "size=" + size +
              ", filePos=" + filePos +
              ", section=" + section +
              ", nelems=" + nelems +
              ", isVlen=" + isVlen +
              ", isDeflate=" + isDeflate;
    }
  }

  public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
    List<DataStorage> storage = (List<DataStorage>) v.getSPobject();
    ByteBuffer result = null;

    for (DataStorage dataStorage : storage) {
      if (dataStorage.isVlen)
        return readVlenData(v, section, dataStorage);

      if (dataStorage.sdata != null) {
        assert (v instanceof Structure);
        return readStructureData((Structure) v, section, dataStorage);
      }

      if (dataStorage.section.intersects(section)) { // LOOK WRONG
        raf.seek(dataStorage.filePos);
        byte[] data = new byte[dataStorage.size];
        raf.readFully(data);

        if (dataStorage.isDeflate) {
          ByteArrayInputStream bin = new ByteArrayInputStream(data);
          InflaterInputStream in = new InflaterInputStream(bin);
          ByteArrayOutputStream bout = new ByteArrayOutputStream(data.length * 7);
          IO.copy(in, bout);
          byte[] resultb = bout.toByteArray();
          result = ByteBuffer.wrap(resultb); // look - an extra copy !! override ByteArrayOutputStream to fix
          if (debug) System.out.printf(" uncompressedLen header=%d actual=%d%n", dataStorage.uncompressedLen , resultb.length);
          result.order(dataStorage.bo);

        } else {
          result = ByteBuffer.wrap(data);
          result.order(dataStorage.bo);
        }
      }
    }

    if (result == null) return null;

    return Array.factory(v.getDataType(), v.getShape(), result);
    //return dataArray.sectionNoReduce(section.getRanges());
  }

  private Array readStructureData(Structure v, Section section, DataStorage dataStorage) throws IOException, InvalidRangeException {
    ByteBuffer bb = dataStorage.sdata.getData().asReadOnlyByteBuffer();
    bb.order(dataStorage.bo);
    StructureMembers sm = v.makeStructureMembers();
    ArrayStructureBB.setOffsets(sm);
    // StructureMembers members, int[] shape, ByteBuffer bbuffer, int offset
    ArrayStructureBB all =  new ArrayStructureBB(sm, v.getShape(), bb, 0);
    return ArrayStructureBBsection.factory(all, section);
  }

  // lOOK probably desnt work
  private Array readVlenData(Variable v, Section section, DataStorage dataStorage) throws IOException, InvalidRangeException {
    raf.seek(dataStorage.filePos);
    int nelems = readVInt(raf);
    Array[] result = new Array[nelems];

    for (int elem = 0; elem < nelems; elem++) {
      int dsize = readVInt(raf);
      byte[] data = new byte[dsize];
      raf.readFully(data);
      Array dataArray = Array.factory(v.getDataType(), (int[]) null, ByteBuffer.wrap(data));
      result[elem] = dataArray;
    }
    // return Array.makeObjectArray(v.getDataType(), result[0].getClass(), new int[]{nelems}, result);
    return Array.makeVlenArray(new int[]{nelems}, result);
    //return dataArray.section(section.getRanges());
  }

  private int readVInt(RandomAccessFile raf) throws IOException {
    byte b = (byte) raf.read();
    int i = b & 0x7F;
    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
      b = (byte) raf.read();
      i |= (b & 0x7F) << shift;
    }
    return i;
  }

  private boolean readAndTest(RandomAccessFile raf, byte[] test) throws IOException {
    byte[] b = new byte[test.length];
    raf.readFully(b);
    return test(b, test);
  }

  private boolean test(byte[] bread, byte[] test) throws IOException {
    if (bread.length != test.length) return false;
    for (int i = 0; i < bread.length; i++)
      if (bread[i] != test[i]) return false;
    return true;
  }

  ///////////////////////////////////////////////////////////////////////////////
  // lower level interface for debugging
  // optionally read in all messages, return as List<NcMess>

  public void openDebug(RandomAccessFile raf, NetcdfFile ncfile, List<NcsMess> ncm) throws IOException {
    raf.seek(0);
    long pos = raf.getFilePointer();

    if (!readAndTest(raf, NcStream.MAGIC_START)) {
      if (ncm != null) {
        ncm.add(new NcsMess(pos, 0, "MAGIC_START missing - abort"));
        return;
      }
      throw new IOException("Data corrupted on " + raf.getLocation());
    }
    if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_START"));

    pos = raf.getFilePointer();
    if (!readAndTest(raf, NcStream.MAGIC_HEADER)) {
      if (ncm != null) {
        ncm.add(new NcsMess(pos, 0, "MAGIC_HEADER missing - abort"));
        return;
      }
      throw new IOException("Data corrupted on " + ncfile.getLocation());
    }
    if (ncm != null) ncm.add(new NcsMess(pos, 4,  "MAGIC_HEADER"));

    // assume for the moment it always starts with one header message
    pos = raf.getFilePointer();
    int msize = readVInt(raf);
    byte[] m = new byte[msize];
    raf.readFully(m);
    NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);
    if (ncm != null) ncm.add(new NcsMess(pos, msize, proto));
    version = proto.getVersion();

    NcStreamProto.Group root = proto.getRoot();
    NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
    ncfile.finish();

    // then we have a stream of data messages with a final END or ERR
    while (!raf.isAtEndOfFile()) {
      pos = raf.getFilePointer();
      byte[] b = new byte[4];
      raf.readFully(b);
      if (test(b, NcStream.MAGIC_END)) {
        if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_END"));
        break;
      }

      if (test(b, NcStream.MAGIC_ERR)) {
        int esize = readVInt(raf);
        byte[] dp = new byte[esize];
        raf.readFully(dp);
        NcStreamProto.Error error = NcStreamProto.Error.parseFrom(dp);
        if (ncm != null) ncm.add(new NcsMess(pos, esize, error.getMessage()));
        break; // assume broken now ?
      }

      if (!test(b, NcStream.MAGIC_DATA)) {
        if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_DATA missing - abort"));
        break;
      }
      if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_DATA"));

      // data messages
      pos = raf.getFilePointer();
      int psize = readVInt(raf);
      byte[] dp = new byte[psize];
      raf.readFully(dp);
      NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
      ByteOrder bo = NcStream.decodeDataByteOrder(dproto); // LOOK not using bo !!

      Variable v = ncfile.findVariable(dproto.getVarName());
      if (v == null) {
        System.out.printf(" ERR cant find var %s%n%s%n", dproto.getVarName(), dproto);
      }
      if (debug) System.out.printf(" dproto = %s for %s%n", dproto, v.getShortName());
      if (ncm != null) ncm.add(new NcsMess(pos, psize, dproto));
      List<DataStorage> storage;
      if (v != null) {
        storage = (List<DataStorage>) v.getSPobject(); // LOOK could be an in memory Rtree using section
        if (storage == null) {
          storage = new ArrayList<>();
          v.setSPobject(storage);
        }
      } else
        storage = new ArrayList<>(); // barf

      // version < 3
      if (dproto.getDataType() == NcStreamProto.DataType.STRUCTURE) {
        pos = raf.getFilePointer();
        msize = readVInt(raf);
        m = new byte[msize];
        raf.readFully(m);
        NcStreamProto.StructureData sdata = NcStreamProto.StructureData.parseFrom(m);
        DataStorage dataStorage = new DataStorage(msize, pos, dproto);
        dataStorage.sdata = sdata;
        if (ncm != null) ncm.add(new NcsMess(dataStorage.filePos, msize, sdata));
        storage.add(dataStorage);

      } else if (dproto.getVdata()) {
        DataStorage dataStorage = new DataStorage(0, raf.getFilePointer(), dproto);
        int nelems = readVInt(raf);
        int totalSize = 0;
        for (int i = 0; i < nelems; i++) {
          int dsize = readVInt(raf);
          totalSize += dsize;
          raf.skipBytes(dsize);
        }
        dataStorage.nelems = nelems;
        dataStorage.size = totalSize;
        if (ncm != null) ncm.add(new NcsMess(dataStorage.filePos, totalSize, dataStorage));
        storage.add(dataStorage);

      } else {  // regular data
        int dsize = readVInt(raf);
        DataStorage dataStorage = new DataStorage(dsize, raf.getFilePointer(), dproto);
        if (ncm != null) ncm.add(new NcsMess(dataStorage.filePos, dsize, dataStorage));
        storage.add(dataStorage);
        raf.skipBytes(dsize);

      }
    }
  }

  public class NcsMess {
    public long filePos;
    public int len;
    public int nelems;
    public Object what;
    public DataType dataType;
    public String varName;

    public NcsMess(long filePos, int len, Object what) {
      this.filePos = filePos;
      this.len = len;
      this.what = what;
      if (what instanceof NcStreamProto.Data) {
        NcStreamProto.Data dataMess = (NcStreamProto.Data) what;
        this.dataType = NcStream.convertDataType(dataMess.getDataType());
        this.varName = dataMess.getVarName();
        Section s = NcStream.decodeSection(dataMess.getSection());
        if (s != null)
          this.nelems = (int) s.computeSize();
      }
      else if (what instanceof DataStorage)
        this.nelems = ((DataStorage) what).nelems;
    }

    public String showDeflate() {
      if (!(what instanceof NcStreamIosp.DataStorage))
        return "Must select a NcStreamIosp.DataStorage object";

      Formatter f = new Formatter();
      try {
        NcStreamIosp.DataStorage dataStorage = (NcStreamIosp.DataStorage) what;

        raf.seek(dataStorage.filePos);
        byte[] data = new byte[dataStorage.size];
        raf.readFully(data);

        ByteArrayInputStream bin = new ByteArrayInputStream(data);
        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        DeflaterOutputStream dout = new DeflaterOutputStream(bout);
        IO.copy(bin, dout);
        dout.close();
        int deflatedSize = bout.size();
        float ratio = ((float)data.length) / deflatedSize;
        f.format("Original size = %d bytes, deflated = %d bytes ratio = %f %n", data.length, deflatedSize, ratio);
        return f.toString();

      } catch (IOException e) {
        e.printStackTrace();
        return e.getMessage();
      }
    }

  }

}


/*
 * Copyright (c) 1998-2017 University Corporation for Atmospheric Research/Unidata
 * See LICENSE.txt for license information.
 */

package thredds.inventory;

import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.Namespace;
import org.jdom2.input.SAXBuilder;
import org.jdom2.output.XMLOutputter;

import thredds.client.catalog.Catalog;
import ucar.nc2.util.AliasTranslator;
import ucar.nc2.util.URLnaming;

import java.io.*;
import java.net.URL;
import java.util.*;

/**
 * Convert NcML into a CollectionManager.
 * Must have aggregation and scan/scanFmrc element.
 * Tracks inner and outer ncml modifications.
 *
 * @author caron
 * @since Feb 24, 2010
 */
public class NcmlCollectionReader {
  // static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(NcmlCollectionReader.class);

  private static final boolean debugURL = false, debugXML = false, showParsedXML = false;
  //private static final boolean validate = false;

  static private final Namespace ncNSHttp = thredds.client.catalog.Catalog.ncmlNS;
  static private final Namespace ncNSHttps = thredds.client.catalog.Catalog.ncmlNSHttps;
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(NcmlCollectionReader.class);

  private Namespace ncmlNS;

  /**
   * Read an NcML file from a String, and construct a NcmlCollectionReader from its scan or scanFmrc element.
   *
   * @param ncmlString the NcML to construct the reader from
   * @param errlog put error messages here
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  static public NcmlCollectionReader readNcML(String ncmlString, Formatter errlog) throws IOException {
      StringReader reader = new StringReader(ncmlString);

      org.jdom2.Document doc;
      try {
        SAXBuilder builder = new SAXBuilder();
        if (debugURL) System.out.println(" NetcdfDataset NcML String = <" + ncmlString + ">");
        doc = builder.build(new StringReader(ncmlString));
      } catch (JDOMException e) {
        throw new IOException(e.getMessage());
      }
      if (debugXML) System.out.println(" SAXBuilder done");

      return readXML(doc, errlog, null);
  }

  /**
   * Read an NcML file from a URL location, and construct a NcmlCollectionReader from its scan or scanFmrc element.
   *
   * @param ncmlLocation the URL location string of the NcML document
   * @param errlog put error messages here
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  static public NcmlCollectionReader open(String ncmlLocation, Formatter errlog) throws IOException {
    if (!ncmlLocation.startsWith("http:") && !ncmlLocation.startsWith("file:"))
      ncmlLocation = "file:" + ncmlLocation;

    URL url = new URL(ncmlLocation);

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      if (debugURL) System.out.println(" NetcdfDataset URL = <" + url + ">");
      doc = builder.build(url);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML) System.out.println(" SAXBuilder done");

    return readXML(doc, errlog, ncmlLocation);
  }

  static private NcmlCollectionReader readXML(org.jdom2.Document doc, Formatter errlog, String ncmlLocation) {
    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element netcdfElem = doc.getRootElement();
    Namespace myNS = netcdfElem.getNamespace(); // detect incorrect namespace

    if (!myNS.equals(Catalog.ncmlNS) && !myNS.equals(Catalog.ncmlNSHttps)) {
      errlog.format("Incorrect namespace specified in NcML= %s must be %s%n or %s%n", myNS.getURI(),
              Catalog.ncmlNS.getURI(), Catalog.ncmlNSHttps.getURI());
      return null;
    }

    Element aggElem = netcdfElem.getChild("aggregation", myNS);
    if (aggElem == null) {
      errlog.format("NcML must have aggregation element");
      return null;
    }

    String type = aggElem.getAttributeValue("type");
    if (!type.equals("forecastModelRunCollection") && !type.equals("forecastModelRunSingleCollection") &&
        !type.equals("fmrc")) {
       errlog.format("NcML aggregation must be of type fmrc");
      return null;
    }

    return new NcmlCollectionReader(ncmlLocation, netcdfElem);
  }

  //////////////////////////////////////////////////////////////////
  private MFileCollectionManager datasetManager;
  private boolean hasInner, hasOuter;
  private Element netcdfElem, aggElem;

  NcmlCollectionReader(String ncmlLocation, Element netcdfElem) {

    if (netcdfElem.equals(Catalog.ncmlNSHttps)) {
      this.ncmlNS = Catalog.ncmlNSHttps;
    } else {
      this.ncmlNS = Catalog.ncmlNS;
    }
    Element aggElem = netcdfElem.getChild("aggregation", ncmlNS);
    String recheck = aggElem.getAttributeValue("recheckEvery");

    // get the aggregation/scan element
    Element scanElem = aggElem.getChild("scan", ncmlNS);
    if (scanElem == null) scanElem = aggElem.getChild("scanFmrc", ncmlNS);

    if (scanElem == null) {
      // no directory scan going on here - look for explicitly named datasets
      Map<String, String> realLocationRunTimeMap = new HashMap<>();
      List<String> realLocationList = new ArrayList<>();
      java.util.List<Element> ncList = aggElem.getChildren("netcdf", ncmlNS);
      for (Element netcdfElemNested : ncList) {
        String location = netcdfElemNested.getAttributeValue("location");
        if (location == null)
          location = netcdfElemNested.getAttributeValue("url");
        if (location != null)
          location = AliasTranslator.translateAlias(location);

        String runTime = netcdfElemNested.getAttributeValue("coordValue");
        if (runTime == null) {
          Formatter f = new Formatter();
          f.format("runtime must be explicitly defined for each netcdf element using the attribute coordValue");
          log.error(f.toString());
        }

        String realLocation = URLnaming.resolveFile(ncmlLocation, location);
        realLocationRunTimeMap.put(realLocation, runTime);
      }
      datasetManager = MFileCollectionManager.openWithRecheck(ncmlLocation, recheck);
      datasetManager.setFilesAndRunDate(realLocationRunTimeMap);

    } else {
      String dirLocation = scanElem.getAttributeValue("location");
      dirLocation = URLnaming.resolve(ncmlLocation, dirLocation); // possible relative location

      String regexpPatternString = scanElem.getAttributeValue("regExp");
      String suffix = scanElem.getAttributeValue("suffix");
      String subdirs = scanElem.getAttributeValue("subdirs");
      String olderThan = scanElem.getAttributeValue("olderThan");

      datasetManager = MFileCollectionManager.openWithRecheck(ncmlLocation, recheck);
      datasetManager.addDirectoryScan(dirLocation, suffix, regexpPatternString, subdirs, olderThan, null);

      String dateFormatMark = scanElem.getAttributeValue("dateFormatMark");
      DateExtractor dateExtractor = null;
      if (dateFormatMark != null)
        dateExtractor = new DateExtractorFromName(dateFormatMark, true);
      else {
        String runDateMatcher = scanElem.getAttributeValue("runDateMatcher");
        if (runDateMatcher != null)
          dateExtractor = new DateExtractorFromName(runDateMatcher, false);
      }
      datasetManager.setDateExtractor(dateExtractor);

    }

    hasOuter = hasMods(netcdfElem);
    hasInner = hasMods(aggElem);

    if (hasOuter)
      this.netcdfElem = netcdfElem;
    if (hasInner)
      this.aggElem = aggElem;
  }

  private boolean hasMods(Element elem) {
    if (elem.getChildren("attribute", ncmlNS).size() > 0) return true;
    if (elem.getChildren("variable", ncmlNS).size() > 0) return true;
    if (elem.getChildren("dimension", ncmlNS).size() > 0) return true;
    if (elem.getChildren("group", ncmlNS).size() > 0) return true;
    if (elem.getChildren("remove",ncmlNS).size() > 0) return true;
    return false;
  }

  public Element getNcmlOuter() {
    return netcdfElem;
  }

  public Element getNcmlInner() {
    return aggElem;
  }

  public CollectionManager getCollectionManager() {
    return datasetManager;
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.unidata.util;

import ucar.nc2.constants.CDM;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.StringTokenizer;

/**
 * Static String and StringBuilder utilities
 *
 * @author caron
 * @since 7/29/11
 */
public class StringUtil2 {

  /**
   * Replace any char not alphanumeric or in allowChars by replaceChar.
   *
   * @param x           operate on this string
   * @param allowChars  these are ok.
   * @param replaceChar thar char to replace
   * @return resulting string.
   */
  static public String allow(String x, String allowChars, char replaceChar) {
    boolean ok = true;
    for (int pos = 0; pos < x.length(); pos++) {
      char c = x.charAt(pos);
      if (!(Character.isLetterOrDigit(c) || (0 <= allowChars.indexOf(c)))) {
        ok = false;
        break;
      }
    }
    if (ok)
      return x;

    // gotta do it
    StringBuilder sb = new StringBuilder(x);
    for (int pos = 0; pos < sb.length(); pos++) {
      char c = sb.charAt(pos);
      if (Character.isLetterOrDigit(c) || (0 <= allowChars.indexOf(c))) {
        continue;
      }

      sb.setCharAt(pos, replaceChar);
    }

    return sb.toString();
  }

  /**
   * Break the given text into lines, respecting word boundaries (blank space).
   *
   * @param text     The text to convert
   * @param insert   break to insert
   * @param lineSize line size to insert at
   * @return The text with added br tags.
   */
  public static String breakTextAtWords(String text, String insert, int lineSize) {
    StringBuilder buff = new StringBuilder();
    StringTokenizer stoker = new StringTokenizer(text);
    int lineCount = 0;
    while (stoker.hasMoreTokens()) {
      String tok = stoker.nextToken();
      if (tok.length() + lineCount >= lineSize) {
        buff.append(insert);
        lineCount = 0;
      }
      buff.append(tok);
      buff.append(" ");
      lineCount += tok.length() + 1;
    }

    return buff.toString();
  }

  /**
   * Delete any non-printable characters
   *
   * @param h byte array
   * @return cleaned up string
   */
  public static String cleanup(byte[] h) {
    byte[] bb = new byte[h.length];
    int count = 0;
    for (byte b : h) {
      if (b >= 32 && b < 127)
        bb[count++] = b;
    }
    return new String(bb, 0, count, CDM.utf8Charset);
  }

  public static String cleanup(String s) {
    if (s == null) return null;
    return cleanup(s.getBytes(CDM.utf8Charset));
  }

  /**
   * Remove any char not alphanumeric or in okChars.
   *
   * @param x       filter this string
   * @param okChars these are ok.
   * @return filtered string.
   */
  static public String filter(String x, String okChars) {
    boolean ok = true;
    for (int pos = 0; pos < x.length(); pos++) {
      char c = x.charAt(pos);
      if (!(Character.isLetterOrDigit(c) || (0 <= okChars.indexOf(c)))) {
        ok = false;
        break;
      }
    }
    if (ok) {
      return x;
    }

    // gotta do it
    StringBuilder sb = new StringBuilder(x.length());
    for (int pos = 0; pos < x.length(); pos++) {
      char c = x.charAt(pos);
      if (Character.isLetterOrDigit(c) || (0 <= okChars.indexOf(c))) {
        sb.append(c);
      }
    }

    return sb.toString();
  }

  /**
   * Remove all but printable ascii
   *
   * @param s filter this string
   * @return filtered string.
   */
  static public String filter7bits(String s) {
    if (s == null) return null;
    char[] bo = new char[s.length()];
    int count = 0;
    for (int i = 0; i < s.length(); i++) {
      char c = s.charAt(i);
      if ((c < 128) && (c > 31) || ((c == '\n') || (c == '\t'))) {
        bo[count++] = c;
      }
    }

    return new String(bo, 0, count);
  }

  // remove leading and trailing blanks
  // remove control characters (< 0x20)
  // transform "/" to "_"
  // transform embedded space to "_"
  static public String makeValidCdmObjectName(String name) {
    name = name.trim();
    // common case no change
    boolean ok = true;
    for (int i = 0; i < name.length(); i++) {
      int c = name.charAt(i);
      if (c < 0x20) ok = false;
      if (c == '/') ok = false;
      if (c == ' ') ok = false;
      if (!ok) break;
    }
    if (ok) return name;

    StringBuilder sbuff = new StringBuilder(name.length());
    for (int i = 0, len = name.length(); i < len; i++) {
      int c = name.charAt(i);
      if ((c == '/') || (c == ' '))
        sbuff.append('_');
      else if (c >= 0x20)
        sbuff.append((char) c);
    }
    return sbuff.toString();
  }

  /**
   * Count number of chars that match in two strings, starting from front.
   *
   * @param s1 compare this string
   * @param s2 compare this string
   * @return number of matching chars, starting from first char
   */
  static public int match(String s1, String s2) {
    int i = 0;
    while ((i < s1.length()) && (i < s2.length())) {
      if (s1.charAt(i) != s2.charAt(i)) {
        break;
      }
      i++;
    }
    return i;
  }

  /**
   * Left pad the given value with zeros up to the number of digits
   *
   * @param value     The value.
   * @param numDigits number of digits
   * @return The String  represenation of the value, padded with
   * leading "0"-s if value < 10E(numDigits-1)
   */
  public static String padZero(int value, int numDigits) {
    return padLeft(String.valueOf(value), numDigits, "0");
  }

  /**
   * Pad the given string with spaces on the left up to the given length.
   *
   * @param s             String to pad
   * @param desiredLength ending length
   * @return padded String
   */
  public static String padLeft(String s, int desiredLength) {
    return padLeft(s, desiredLength, " ");
  }

  /**
   * Pad the given string with padString on the left up to the given length.
   *
   * @param s             String to pad
   * @param desiredLength ending length
   * @param padString     String to pad with (e.g, " ")
   * @return padded String
   */
  public static String padLeft(String s, int desiredLength,
                               String padString) {
    while (s.length() < desiredLength) {
      s = padString + s;
    }
    return s;
  }

  /**
   * Pad the given string with spaces on the right up to the given length.
   *
   * @param s             String to pad
   * @param desiredLength ending length
   * @return padded String
   */
  public static String padRight(String s, int desiredLength) {
    return padRight(s, desiredLength, " ");
  }

  /**
   * Pad the given string with padString on the right up to the given length.
   *
   * @param s             String to pad
   * @param desiredLength ending length
   * @param padString     String to pad with (e.g, " ")
   * @return padded String
   */
  public static String padRight(String s, int desiredLength,
                                String padString) {
    StringBuilder ret = new StringBuilder(s);
    while (ret.length() < desiredLength) {
      ret.append(padString);
    }
    return ret.toString();
  }

  /**
   * Remove all occurrences of the substring sub in the string s.
   *
   * @param s   operate on this string
   * @param sub remove all occurrences of this substring.
   * @return result with substrings removed
   */
  static public String remove(String s, String sub) {
    int len = sub.length();
    int pos;
    while (0 <= (pos = s.indexOf(sub))) {
      s = s.substring(0, pos) + s.substring(pos + len);
    }
    return s;
  }

  /**
   * Remove all occurrences of the character c in the string s.
   *
   * @param s operate on this string
   * @param c remove all occurrences of this character.
   * @return result with any character c removed
   */
  static public String remove(String s, int c) {
    if (0 > s.indexOf(c)) {  // none
      return s;
    }

    StringBuilder buff = new StringBuilder(s);
    int i = 0;
    while (i < buff.length()) {
      if (buff.charAt(i) == c) {
        buff.deleteCharAt(i);
      } else {
        i++;
      }
    }
    return buff.toString();
  }

  /**
   * Remove all occurrences of the character c at the end of s.
   *
   * @param s operate on this string
   * @param c remove all occurrences of this character that are at the end of the string.
   * @return result with any character c removed
   */
  static public String removeFromEnd(String s, int c) {
    if (0 > s.indexOf(c))   // none
      return s;

    int len = s.length();
    while ((s.charAt(len - 1) == c) && (len > 0))
      len--;

    if (len == s.length())
      return s;
    return s.substring(0, len);
  }

  static public String removeFromEnd(String s, String suffix) {
    if (s.endsWith(suffix))
      return s.substring(0, s.length() - suffix.length());

    return s;
  }

  /**
   * Remove any whitespace (ie., Character.isWhitespace) from the input string.
   *
   * @param inputString The string to remove the whitespace.
   * @return The whitespaceless result.
   */
  public static String removeWhitespace(String inputString) {
    StringBuilder sb = new StringBuilder();
    char[] chars = inputString.toCharArray();
    for (char c : chars) {
      if (Character.isWhitespace(c)) {
        continue;
      }
      sb.append(c);
    }
    return sb.toString();
  }

  /**
   * Collapse continuous whitespace into one single " ".
   *
   * @param s operate on this string
   * @return result with collapsed whitespace
   */
  static public String collapseWhitespace(String s) {
    int len = s.length();
    StringBuilder b = new StringBuilder(len);
    for (int i = 0; i < len; i++) {
      char c = s.charAt(i);
      if (!Character.isWhitespace(c)) {
        b.append(c);
      } else {
        b.append(' ');
        while ((i + 1 < len) && Character.isWhitespace(s.charAt(i + 1))) {
          i++;  /// skip further whitespace
        }
      }
    }
    return b.toString();
  }

  /**
   * Replace any char "out" in s with "in".
   *
   * @param s   string to replace
   * @param out replace this character
   * @param in  with this string
   * @return modified string if needed
   */
  static public String replace(String s, char out, String in) {
    if (s.indexOf(out) < 0) {
      return s;
    }

    // gotta do it
    StringBuilder sb = new StringBuilder(s);
    replace(sb, out, in);
    return sb.toString();
  }

  /**
   * Replace all occurrences of any char in replaceChar with corresponding String in replaceWith
   *
   * @param x           operate on this string
   * @param replaceChar get rid of these
   * @param replaceWith replace with these
   * @return resulting string
   */
  static public String replace(String x, char[] replaceChar, String[] replaceWith) {
    // common case no replacement
    boolean ok = true;
    for (char aReplaceChar : replaceChar) {
      int pos = x.indexOf(aReplaceChar);
      ok = (pos < 0);
      if (!ok)
        break;
    }
    if (ok)
      return x;

    // gotta do it
    StringBuilder sb = new StringBuilder(x);
    for (int i = 0; i < replaceChar.length; i++) {
      int pos = x.indexOf(replaceChar[i]);
      if (pos >= 0) {
        replace(sb, replaceChar[i], replaceWith[i]);
      }
    }

    return sb.toString();
  }

  /**
   * Replaces all occurrences of "pattern" in "string" with "value"
   *
   * @param string  string to munge
   * @param pattern pattern to replace
   * @param value   replacement value
   * @return munged string
   */
  public static String replace(String string, String pattern, String value) {
    if (pattern.length() == 0)
      return string;

    if (!string.contains(pattern)) return string;

    // ok gotta do it
    StringBuilder returnValue = new StringBuilder();
    int patternLength = pattern.length();
    while (true) {
      int idx = string.indexOf(pattern);
      if (idx < 0)
        break;

      returnValue.append(string.substring(0, idx));
      if (value != null)
        returnValue.append(value);

      string = string.substring(idx + patternLength);
    }
    returnValue.append(string);
    return returnValue.toString();
  }

  /**
   * Replace all occurrences of orgReplace with orgChar; inverse of replace().
   *
   * @param x          operate on this string
   * @param orgReplace get rid of these
   * @param orgChar    replace with these
   * @return resulting string
   */
  static public String unreplace(String x, String[] orgReplace, char[] orgChar) {
    // common case no replacement
    boolean ok = true;
    for (String anOrgReplace : orgReplace) {
      int pos = x.indexOf(anOrgReplace);
      ok = (pos < 0);
      if (!ok) break;
    }
    if (ok)
      return x;

    // gotta do it
    StringBuilder result = new StringBuilder(x);
    for (int i = 0; i < orgReplace.length; i++) {
      int pos = result.indexOf(orgReplace[i]);
      if (pos >= 0) {
        unreplace(result, orgReplace[i], orgChar[i]);
      }
    }

    return result.toString();
  }

  /**
   * Find all occurrences of the "match" in original, and substitute the "subst" string.
   *
   * @param original starting string
   * @param match    string to match
   * @param subst    string to substitute
   * @return a new string with substitutions
   */
  static public String substitute(String original, String match, String subst) {
    String s = original;
    int pos;
    while (0 <= (pos = s.indexOf(match))) {
      StringBuilder sb = new StringBuilder(s);
      s = sb.replace(pos, pos + match.length(), subst).toString();
    }
    return s;
  }

  /**
   * Escape any char not alphanumeric or in okChars.
   * Escape by replacing char with %xx (hex).
   *
   * @param x       escape this string
   * @param okChars these are ok.
   * @return equivilent escaped string.
   */
  static public String escape(String x, String okChars) {
    StringBuilder newname = new StringBuilder();
    for (char c : x.toCharArray()) {
      if (c == '%') {
        newname.append("%%");
      } else if (!Character.isLetterOrDigit(c) && okChars.indexOf(c) < 0) {
        newname.append('%');
        newname.append(Integer.toHexString((0xFF & (int) c)));
      } else
        newname.append(c);
    }
    return newname.toString();
  }

  /**
   * This finds any '%xx' and converts to the equivalent char. Inverse of
   * escape().
   *
   * @param x operate on this String
   * @return original String.
   */
  static public String unescape(String x) {
    if (x.indexOf('%') < 0) {
      return x;
    }

    // gotta do it
    char[] b = new char[2];
    StringBuilder sb = new StringBuilder(x);
    for (int pos = 0; pos < sb.length(); pos++) {
      char c = sb.charAt(pos);
      if (c != '%') {
        continue;
      }
      if (pos >= sb.length() - 2) { // malformed - should be %xx
        return x;
      }
      b[0] = sb.charAt(pos + 1);
      b[1] = sb.charAt(pos + 2);
      int value;
      try {
        value = Integer.parseInt(new String(b), 16);
      } catch (NumberFormatException e) {
        continue;   // not a hex number
      }
      c = (char) value;
      sb.setCharAt(pos, c);
      sb.delete(pos + 1, pos + 3);
    }

    return sb.toString();
  }

  /**
   * Split a string on one or more whitespace.
   * Cover for String.split, because who can remember regexp?
   *
   * @param source split this string
   * @return tokens that were seperated by whitespace
   */
  public static String[] splitString(String source) {
    return source.trim().split("\\s+"); // Separated by "whitespace"
  }

  /**
   * Find all occurences of match strings in original, and substitute the corresponding
   * subst string.
   *
   * @param original starting string
   * @param match    array of strings to match
   * @param subst    array of strings to substitute
   * @return a new string with substitutions
   */
  static public String substitute(String original, String[] match, String[] subst) {

    boolean ok = true;
    for (String aMatch : match) {
      if (original.contains(aMatch)) {
        ok = false;
        break;
      }
    }
    if (ok) {
      return original;
    }

    // gotta do it;
    StringBuilder sb = new StringBuilder(original);
    for (int i = 0; i < match.length; i++) {
      substitute(sb, match[i], subst[i]);
    }
    return sb.toString();
  }

  static public List<String> getTokens(String fullString, String sep) throws Exception {

    List<String> strs = new ArrayList<>();
    if (sep != null) {
      int sepLength = sep.length();
      switch (sepLength) {
        case 0:
          String[] tokens = splitString(fullString);   // default to use white space if separator is ""
          strs = Arrays.asList(tokens);
          break;

        case 1:
          StringTokenizer tokenizer = new StringTokenizer(fullString, sep);       // maybe use StreamTokenizer?
          while (tokenizer.hasMoreTokens())
            strs.add(tokenizer.nextToken());
          break;

        default:
          String remainderString = fullString;    // multicharacter separator
          int location = remainderString.indexOf(sep);
          while (location != -1) {    // watch out for off-by-one errors on the string splitting indices!!!
            if (location == 0) {    // remainderString starts with the separator, cut it off
              remainderString = remainderString.substring(location + sepLength);
              location = remainderString.indexOf(sep);
            } else {
              String token = remainderString.substring(0, location); // pull the token off the front of the string
              strs.add(token);    // add the token to our list
              remainderString = remainderString.substring(location + sepLength); // cut out both the token and the separator
              location = remainderString.indexOf(sep);
            }
          }  //close while loop
          if (remainderString.length() > 0) strs.add(remainderString);    // add the last token, post last separator
      }  //close switch (sepLength)
    } else {  // default to white space separator if sep is null
      String[] tokens = splitString(fullString);
      strs = Arrays.asList(tokens);
    }
    if (strs.size() == 0) strs.add(""); // maybe thrown an exception instead?  return null?
    return strs;
  }

  ////////////////////////////////////////////////////
  // StringBuilder

  /**
   * Remove any of the characters in out from sb
   *
   * @param sb  the StringBuilder
   * @param out get rid of any of these characters
   */
  static public void remove(StringBuilder sb, String out) {
    int i = 0;
    while (i < sb.length()) {
      int c = sb.charAt(i);
      boolean ok = true;
      for (int j = 0; j < out.length(); j++) {
        if (out.charAt(j) == c) {
          sb.delete(i, i + 1);
          ok = false;
          break;
        }
      }
      if (ok) i++;
    }
  }

  /**
   * Replace any char "out" in sb with String "in".
   *
   * @param sb  StringBuilder to replace
   * @param out repalce this character
   * @param in  with this string
   */
  static public void replace(StringBuilder sb, char out, String in) {
    for (int i = 0; i < sb.length(); i++) {
      if (sb.charAt(i) == out) {
        sb.replace(i, i + 1, in);
        i += in.length() - 1;
      }
    }
  }

  /**
   * Replace any String "out" in sb with char "in".
   *
   * @param sb  StringBuilder to replace
   * @param out repalce this String
   * @param in  with this char
   */
  static public void unreplace(StringBuilder sb, String out, char in) {
    int pos;
    while (0 <= (pos = sb.indexOf(out))) {
      sb.setCharAt(pos, in);
      sb.delete(pos + 1, pos + out.length());
    }
  }

  /**
   * Replace any of the characters from out with corresponding character from in
   *
   * @param sb  the StringBuilder
   * @param out get rid of any of these characters
   * @param in  replacing with the character at same index
   */
  static public void replace(StringBuilder sb, String out, String in) {
    for (int i = 0; i < sb.length(); i++) {
      int c = sb.charAt(i);
      for (int j = 0; j < out.length(); j++) {
        if (out.charAt(j) == c)
          sb.setCharAt(i, in.charAt(j));
      }
    }
  }

  /**
   * Find all occurences of the "match" in original, and substitute the "subst" string,
   * directly into the original.
   *
   * @param sbuff starting string buffer
   * @param match string to match
   * @param subst string to substitute
   */
  static public void substitute(StringBuilder sbuff, String match, String subst) {
    int pos, fromIndex = 0;
    int substLen = subst.length();
    int matchLen = match.length();
    while (0 <= (pos = sbuff.indexOf(match, fromIndex))) {
      sbuff.replace(pos, pos + matchLen, subst);
      fromIndex = pos + substLen;  // make sure dont get into an infinite loop
    }
  }

  /**
   * Remove bad char from beginning or end of string
   *
   * @param s operate on this
   * @return trimmed string
   */

  static public String trim(String s, int bad) {
    int len = s.length();
    int st = 0;

    while ((st < len) && (s.charAt(st) == bad)) {
      st++;
    }
    while ((st < len) && (s.charAt(len - 1) == bad)) {
      len--;
    }
    return ((st > 0) || (len < s.length())) ? s.substring(st, len) : s;
  }

  //////////////////////////////////////////////////////////////////////

  private static final char[] htmlIn = {'&', '"', '\'', '<', '>', '\n'};
  private static final String[] htmlOut = {"&", """, "'", "<", ">", "\n<p>"};

  /**
   * @deprecated legacy only, use HtmlEscapers.htmlEscaper()
   */
  static public String quoteHtmlContent(String x) {
    if (x == null) return null;
    return replace(x, htmlIn, htmlOut);
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.featurecollection;

import com.google.common.eventbus.Subscribe;
import org.slf4j.Logger;
import thredds.client.catalog.*;
import thredds.client.catalog.builder.CatalogBuilder;
import thredds.client.catalog.builder.DatasetBuilder;
import thredds.core.AllowedServices;
import thredds.core.StandardService;
import thredds.inventory.*;
import thredds.server.catalog.FeatureCollectionRef;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.GridDataset;
import ucar.nc2.ft.FeatureDatasetPoint;
import ucar.nc2.ft2.coverage.CoverageCollection;
import ucar.nc2.ft2.simpgeometry.SimpleGeometryFeatureDataset;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.util.URLnaming;

import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
import java.util.List;

/**
 * Abstract superclass for Feature Collection Datasets.
 * This is a InvCatalogRef subclass. So the reference is placed in the parent, but
 * the catalog itself isnt constructed until the following call from DataRootHandler.makeDynamicCatalog():
 * match.dataRoot.featCollection.makeCatalog(match.remaining, path, baseURI);
 * <p>
 * The DatasetFeatureCollection object is held in the DataRootManager's FeatureCollectionCache; it may get closed and recreated.
 *
 * @author caron
 * @since Mar 3, 2010
 */
@ThreadSafe
public abstract class InvDatasetFeatureCollection implements Closeable {
  private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InvDatasetFeatureCollection.class);

  static protected final String LATEST_DATASET_CATALOG = "latest.xml";
  static protected final String VARIABLES = "?metadata=variableMap";
  static protected final String FILES = "files";

  // can be changed
  static protected AllowedServices allowedServices;
  static protected String contextName = "/thredds";  // set by TdsInit

  // cant use spring wiring because InvDatasetFeatureCollection not a spring component because depends on catalog config
  static public void setContextName(String c) {
    contextName = c;
  }

  static public void setAllowedServices(AllowedServices _allowedServices) {
    allowedServices = _allowedServices;
  }

  static protected String buildCatalogServiceHref(String path) {
    return contextName + "/catalog/" + path + "/catalog.xml";
  }

  static public InvDatasetFeatureCollection factory(FeatureCollectionRef parent, FeatureCollectionConfig config) {
    InvDatasetFeatureCollection result;
    if (config.type == FeatureCollectionType.FMRC)
      result = new InvDatasetFcFmrc(parent, config);

    else if (config.type == FeatureCollectionType.GRIB1 || config.type == FeatureCollectionType.GRIB2) {
      result = new InvDatasetFcGrib(parent, config);

    } else {
      result = new InvDatasetFcPoint(parent, config);
    }

    return result;
  }

  /////////////////////////////////////////////////////////////////////////////
  // heres how we manage state changes in a thread-safe way
  protected class State {
    // catalog metadata
    protected ThreddsMetadata.VariableGroup vars;
    protected ThreddsMetadata.GeospatialCoverage coverage;
    protected CalendarDateRange dateRange;

    //protected DatasetBuilder top;        // top dataset LOOK why ??
    protected long lastInvChange;        // last time dataset inventory was changed
    protected long lastProtoChange;      // last time proto dataset was changed

    protected State(State from) {
      if (from != null) {
        this.vars = from.vars;
        this.coverage = from.coverage;
        this.dateRange = from.dateRange;
        this.lastProtoChange = from.lastProtoChange;

        //this.top = from.top;
        this.lastInvChange = from.lastInvChange;
      }
    }

    protected State copy() {  // allow override
      return new State(this);
    }
  }

  /////////////////////////////////////////////////////////////////////////////

  // not changed after first call
  protected FeatureCollectionRef parent;
  protected Service orgService, virtualService, latestService, downloadService;

  // from the config catalog
  protected final String name;
  protected final String configPath;
  protected final FeatureCollectionType fcType;
  protected final FeatureCollectionConfig config;
  protected String topDirectory;
  protected MFileCollectionManager datasetCollection; // defines the collection of datasets in this feature collection, actually final NOT USED BY GRIB

  @GuardedBy("lock")
  protected State state;
  @GuardedBy("lock")
  protected boolean first = true;
  protected final Object lock = new Object();

  protected InvDatasetFeatureCollection(FeatureCollectionRef parent, FeatureCollectionConfig config) {
    this.parent = parent;
    this.name = config.name;
    this.configPath = config.path;
    this.fcType = config.type;
    this.config = config;

    makeDefaultServices();

    // this.getLocalMetadataInheritable().setDataType(fcType.getFeatureType());
    logger.info("FeatureCollection added = {}", getConfig());
  }

  protected void makeDefaultServices() {
    latestService = allowedServices.getStandardService(StandardService.resolver);
    downloadService = allowedServices.getStandardService(StandardService.httpServer);

    orgService = parent.getServiceDefault();
    if (orgService == null) {
      String orgServiceName = parent.getServiceNameDefault();
      orgService = allowedServices.findGlobalService(orgServiceName);
    }
    if (orgService == null) {
      orgService = allowedServices.getStandardServices(fcType.getFeatureType());
    }
    if (orgService == null)
      return;

    if (orgService.getType() != ServiceType.Compound) {
      virtualService = orgService;
      return;
    }

    // remove http service
    List<Service> nestedOk = new ArrayList<>();
    for (Service service : orgService.getNestedServices()) {
      if (service.getType() != ServiceType.HTTPServer) {
        nestedOk.add(service);
      }
    }
    virtualService = new Service("VirtualServices", "", ServiceType.Compound.toString(), ServiceType.Compound.getDescription(),
            null, nestedOk, orgService.getProperties(), ServiceType.Compound.getAccessType());
  }

  public void close() {
    if (datasetCollection != null)
      datasetCollection.close();
  }

  //////////////////////////////////////////////////////
  // old school : used by FMRC and Point

  protected void makeCollection() {
    Formatter errlog = new Formatter();
    datasetCollection = new MFileCollectionManager(config, errlog, logger);
    topDirectory = datasetCollection.getRoot();
    String errs = errlog.toString();
    if (errs.length() > 0) logger.warn("MFileCollectionManager parse error = {} ", errs);
  }

  //////////////////////////////////////////////////////
  // called by eventBus, this is where the trigger comes in
  @Subscribe
  public void processEvent(CollectionUpdateEvent event) {
    if (!config.collectionName.equals(event.getCollectionName())) return; // not for me

    try {
      update(event.getType());
    } catch (IOException e) {
      logger.error("Error processing event", e);
    }
  }

  ///////////////////////////////////////////////////

  public void showStatus(Formatter f) {
    try {
      checkState();
      _showStatus(f, false, null);

    } catch (Throwable t) {
      StringWriter sw = new StringWriter(5000);
      t.printStackTrace(new PrintWriter(sw));
      f.format(sw.toString());
    }
  }

  public String showStatusShort(String type) {
    Formatter f = new Formatter();
    try {
      checkState();
      _showStatus(f, true, type);

    } catch (Throwable t) {
      StringWriter sw = new StringWriter(5000);
      t.printStackTrace(new PrintWriter(sw));
      f.format(sw.toString());
    }

    return f.toString();
  }

  protected void _showStatus(Formatter f, boolean summaryOnly, String type) throws IOException {
  }

  protected String getPath() {
    return parent.getUrlPath();
  }

  //////////////////////////////////////////////////////////////////////////////////////////
  // for subclasses

  // localState is synched, may be directly changed
  abstract protected void updateCollection(State localState, CollectionUpdateType force);

  ////////////////////////////////////////////////////////////////////////////////////////////

  protected String getCatalogHref(String what) {
    return buildCatalogServiceHref(configPath + "/" + what);
  }

  // call this first time a request comes in. LOOK could be at construction time i think
  protected void firstInit() {
    // makeDefaultServices();
  }

  /**
   * A request has come in, check that the state has been initialized.
   * this is called from the request thread.
   *
   * @return a copy of the State
   */
  protected State checkState() throws IOException {
    State localState;

    synchronized (lock) {
      if (first) {
        firstInit();
        updateCollection(state, config.updateConfig.updateType);
        // makeDatasetTop(state);
        first = false;
      }
      localState = state.copy();
    }

    return localState;
  }

  /**
   * Collection was changed, update internal objects.
   * called by CollectionUpdater, trigger via handleCollectionEvent, so in a quartz scheduler thread
   *
   * @param force update type
   */
  protected void update(CollectionUpdateType force) throws IOException {  // this may be called from a background thread, or from checkState() request thread
    State localState;

    synchronized (lock) {
      if (first) {
        state = checkState();
        state.lastInvChange = System.currentTimeMillis();
        return;
      }
      // do the update in a local object
      localState = state.copy();
    }

    updateCollection(localState, force);
    // makeDatasetTop(localState);
    localState.lastInvChange = System.currentTimeMillis();

    // switch to live
    synchronized (lock) {
      state = localState;
    }

  }

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  public String getName() {
    return name;
  }

  public String getConfigPath() {
    return configPath;
  }

  public String getLatestFileName() {
    if (config.gribConfig.latestNamer != null) {
      return config.gribConfig.latestNamer;
    } else {
      return "Latest " + name + " File";
    }
  }

  public String getTopDirectoryLocation() {
    return topDirectory;
  }

  public FeatureCollectionConfig getConfig() {
    return config;
  }

  public MCollection getDatasetCollectionManager() {
    return datasetCollection;
  }

  public Logger getLogger() {
    return logger;
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////

  protected String makeFullName(DatasetNode ds) {
    if (ds.getParent() == null) return ds.getName();
    String parentName = makeFullName(ds.getParent());
    if (parentName == null || parentName.length() == 0) return ds.getName();
    return parentName + "/" + ds.getName();
  }

  /**
   * Get one of the catalogs contained in this collection,
   * called by DataRootHandler.makeDynamicCatalog()
   *
   * @param match   match.remaining
   * @param orgPath the path for the request.
   * @param catURI  the base URI for the catalog to be made, used to resolve relative URLs.
   * @return containing catalog
   */
  abstract public CatalogBuilder makeCatalog(String match, String orgPath, URI catURI) throws IOException;

  abstract protected DatasetBuilder makeDatasetTop(URI catURI, State localState) throws IOException;

  /**
   * Make the containing catalog of this feature collection
   * "http://server:port/thredds/catalog/path/catalog.xml"
   *
   * @param catURI     base URI of the request
   * @param localState current state to use
   * @return the top FMRC catalog
   * @throws java.io.IOException         on I/O error
   * @throws java.net.URISyntaxException if path is misformed
   */
  protected CatalogBuilder makeCatalogTop(URI catURI, State localState) throws IOException, URISyntaxException {
    Catalog parentCatalog = parent.getParentCatalog();

    CatalogBuilder topCatalog = new CatalogBuilder();
    topCatalog.setName(makeFullName(parent));
    topCatalog.setVersion(parentCatalog.getVersion());
    topCatalog.setBaseURI(catURI);

    DatasetBuilder top = makeDatasetTop(catURI, localState);
    topCatalog.addDataset(top);
    // topCatalog.addService(StandardServices.latest.getService());  // in case its needed
    topCatalog.addService(virtualService);
    return topCatalog;
  }

  // this catalog lists the individual files comprising the collection.
  protected CatalogBuilder makeCatalogFiles(URI catURI, State localState, List<String> filenames, boolean addLatest) throws IOException {
    Catalog parentCatalog = parent.getParentCatalog();

    CatalogBuilder result = new CatalogBuilder();
    result.setName(makeFullName(parent));
    result.setVersion(parentCatalog.getVersion());
    result.setBaseURI(catURI);
    result.addService(orgService);

    DatasetBuilder top = new DatasetBuilder(null);
    top.transferInheritedMetadata(parent); // make all inherited metadata local
    top.setName(FILES);

    // add Variables, GeospatialCoverage, TimeCoverage
    ThreddsMetadata tmi = top.getInheritableMetadata();
    tmi.set(Dataset.TimeCoverage, null);      // LOOK
    if (localState.coverage != null) {
      tmi.set(Dataset.GeospatialCoverage, localState.coverage);
    }
    tmi.set(Dataset.ServiceName, orgService.getName());
    result.addDataset(top);

    if (addLatest) {
      DatasetBuilder latest = new DatasetBuilder(top);
      latest.setName(getLatestFileName());
      latest.put(Dataset.UrlPath, LATEST_DATASET_CATALOG);
      latest.put(Dataset.Id, LATEST_DATASET_CATALOG);
      latest.put(Dataset.ServiceName, latestService.getName());
      latest.addServiceToCatalog(latestService);
      top.addDataset(latest);
    }

    // sort copy of files
    List<String> sortedFilenames = new ArrayList<>(filenames);
    Collections.sort(sortedFilenames, String.CASE_INSENSITIVE_ORDER);

    // if not increasing (i.e. we WANT newest file listed first), reverse sort
    if (!this.config.getSortFilesAscending()) {
      Collections.reverse(sortedFilenames);
    }

    for (String f : sortedFilenames) {
      if (!f.startsWith(topDirectory))
        logger.warn("File {} doesnt start with topDir {}", f, topDirectory);

      DatasetBuilder ds = new DatasetBuilder(top);

      String fname = f.substring(topDirectory.length() + 1);
      ds.setName(fname);

      String lpath = this.configPath + "/" + FILES + "/" + fname;
      // String lpath = getPath() + "/" + FILES + "/" + fname;
      ds.put(Dataset.UrlPath, lpath);
      ds.put(Dataset.Id, lpath);
      ds.put(Dataset.VariableMapLinkURI, new ThreddsMetadata.UriResolved(makeMetadataLink(lpath, VARIABLES), catURI));

      File file = new File(f);
      ds.put(Dataset.DataSize, file.length());
      top.addDataset(ds);
    }

    return result;
  }

  protected String makeMetadataLink(String datasetName, String metadata) {
    return contextName + "/metadata/" + datasetName + metadata;
  }

  protected ThreddsMetadata.UriResolved makeUriResolved(URI baseURI, String href) {
    try {
      String mapUri = URLnaming.resolve(baseURI.toString(), href);
      return new ThreddsMetadata.UriResolved(href, new URI(mapUri));
    } catch (Exception e) {
      logger.error(" ** Invalid URI= '" + baseURI.toString() + "' href='" + href + "'%n", e);
      return null;
    }
  }

  // called by DataRootHandler.makeDynamicCatalog()
  public CatalogBuilder makeLatest(String matchPath, String reqPath, URI catURI) throws IOException {
    checkState();

    Catalog parentCatalog = parent.getParentCatalog();

    CatalogBuilder result = new CatalogBuilder();
    result.setName(makeFullName(parent));
    result.setVersion(parentCatalog.getVersion());
    result.setBaseURI(catURI);
    result.addService(orgService);

    DatasetBuilder top = new DatasetBuilder(null);
    top.transferInheritedMetadata(parent); // make all inherited metadata local
    top.setName(getLatestFileName());
    top.put(Dataset.ServiceName, orgService.getName());

    result.addDataset(top);

    MFile mfile = datasetCollection.getLatestFile();  // LOOK - assumes dcm is up to date
    if (mfile == null) return null;

    String mpath = mfile.getPath();
    if (!mpath.startsWith(topDirectory))
      logger.warn("File {} doesnt start with topDir {}", mpath, topDirectory);
    String fname = mpath.substring(topDirectory.length() + 1);

    String path = FILES + "/" + fname;
    top.put(Dataset.UrlPath, this.configPath + "/" + path);
    top.put(Dataset.Id, this.configPath + "/" + path);
    String lpath = this.configPath + "/" + path;
    top.put(Dataset.VariableMapLinkURI, new ThreddsMetadata.UriResolved(makeMetadataLink(lpath, VARIABLES), catURI));
    top.put(Dataset.DataSize, mfile.getLength());

    return result;
  }

  /////////////////////////////////////////////////////////////////////////

  /**
   * Get the associated Point Dataset, if any. called by DatasetHandler.openPointDataset()
   *
   * @param matchPath match.remaining
   * @return Grid Dataset, or null if n/a
   * @throws IOException on error
   */
  public FeatureDatasetPoint getPointDataset(String matchPath) throws IOException {
    return null;
  }

  /**
   * Get the associated Grid Dataset, if any. called by DatasetHandler.openGridDataset()
   *
   * @param matchPath match.remaining
   * @return Grid Dataset, or null if n/a
   * @throws IOException on error
   */
  public ucar.nc2.dt.grid.GridDataset getGridDataset(String matchPath) throws IOException {
    return null;
  }

  public CoverageCollection getGridCoverage(String matchPath) throws IOException {
    return null;
  }
  public SimpleGeometryFeatureDataset getSimpleGeometryDataset(String matchPath) throws IOException{
    return null;
  }

  ///////////////////////////////////////////////////////////////////////////////
  // handle individual files

  /**
   * Get the dataset named by the path.
   * called by DatasetHandler.getNetcdfFile()
   *
   * @param matchPath remaining path from match
   * @return requested dataset
   * @throws IOException if read error
   */
  public NetcdfDataset getNetcdfDataset(String matchPath) throws IOException {
    int pos = matchPath.indexOf('/');
    String type = (pos > -1) ? matchPath.substring(0, pos) : matchPath;
    String name = (pos > -1) ? matchPath.substring(pos + 1) : "";

    // this assumes that these are files. also might be remote datasets from a catalog
    if (type.equalsIgnoreCase(FILES)) {
      if (topDirectory == null) return null;

      String filename = new StringBuilder(topDirectory)
              .append(topDirectory.endsWith("/") ? "" : "/")
              .append(name).toString();
      DatasetUrl durl = new DatasetUrl(null, filename);

      return NetcdfDataset.acquireDataset(null, durl, null, -1, null, null); // no enhancement
    }

    GridDataset gds = getGridDataset(matchPath); // LOOK cant be right
    return (gds == null) ? null : (NetcdfDataset) gds.getNetcdfFile();
  }

  // called by DataRootHandler.getCrawlableDatasetAsFile()
  // may have to remove the extra "files" from the path
  // this says that a File URL has to be topDirectory + [FILES/ +] + match.remaining
  public File getFile(String remaining) {
    if (null == topDirectory) return null;
    int pos = remaining.indexOf(FILES);
    StringBuilder fname = new StringBuilder(topDirectory);
    if (!topDirectory.endsWith("/"))
      fname.append("/");
    fname.append((pos > -1) ? remaining.substring(pos + FILES.length() + 1) : remaining);
    return new File(fname.toString());
  }

}