/*
 * Copyright (c) 1998-2017 John Caron and University Corporation for Atmospheric Research/Unidata
 */
package ucar.nc2.ncml;

import org.jdom2.Element;
import thredds.filesystem.MFileOS;
import thredds.filesystem.MFileOS7;
import thredds.inventory.DateExtractor;
import thredds.inventory.DateExtractorFromName;
import thredds.inventory.MFile;
import thredds.inventory.MFileCollectionManager;
import ucar.ma2.Array;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;
import ucar.ma2.Section;
import ucar.nc2.Group;
import ucar.nc2.NetcdfFile;
import ucar.nc2.ProxyReader;
import ucar.nc2.Variable;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.VariableEnhanced;
import ucar.nc2.units.DateFormatter;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.DiskCache2;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.Executor;

/**
 * Superclass for NcML Aggregation.
 *
 * An Aggregation acts as a ProxyReader for VariableDS. That, is it must implement:
 *

 *   public Array read(Variable mainv);
 *   public Array read(Variable mainv, Section section);
 *

 *
 * @author caron
 */

/* May be out of date
 *
Implementation Notes

 *
Caching

 *

 *
Case 1. Explicit list / Scan static directories (recheck=null)
 *

 *
A. AggCaching - keep track of ncoords, coordValues for joinExisting. Read on open, write on close.
 * Could skip scan if cache exists.
 *
B. NetcdfFileCache - write on close if changed (only first time). On sync, recheck = null means wont be reread.
 *

 *
Case 2. Scan dynamic directories (recheck non-null)
 *

 *
A. AggCaching - keep track of ncoords, coordValues for joinExisting. Read on open, write on close.
 * Could skip scan if cache exists, and recheck time not expired.
 *
B. NetcdfFileCache - write on close if changed. On sync, if recheck time, then rescan.
 *

 *

 *
Aggregation Coordinate Variable (aggCoord) Processing

 * Construction:
 *

 *
 The aggregation element is processed first.
 *
 agg.finish() is called.
 *
 If the user has defined the aggCoord in the NcML, it is then processed, overriding whatever the aggregation has constructed.
 * If values are defined, they are cached in the new variable.
 *

 * Data Reading:
 *

 *
 If values are cached, agg.read() is never called.
 *
 Each Dataset may have a coordinate value(s) defined in the NcML coordValue attribute.
 *
 If not, the coordinate value(s) is cached when the dataset is opened.
 *
 agg.read() uses those if they exist, else reads and caches.
 *

 */
public abstract class Aggregation {

  protected enum Type {
    forecastModelRunCollection,
    forecastModelRunSingleCollection,
    joinExisting,
    joinExistingOne, // joinExisting with a DateFormatMark makes it into a joinExistingOne - must have only one coord / file
    joinNew,
    tiled,
    union
  }

  protected enum TypicalDataset {FIRST, RANDOM, LATEST, PENULTIMATE }
  static protected TypicalDataset typicalDatasetMode;

  static protected org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Aggregation.class);
  static protected DiskCache2 diskCache2 = null;

  // this is where  persist() reads/writes files
  static public void setPersistenceCache(DiskCache2 dc) {
    diskCache2 = dc;
    if (diskCache2 != null)
      diskCache2.setAlwaysUseCache(true); // the persistence cache file has same name as the ncml - must put it into the cache else clobber ncml  7/31/2014
  }

  // experimental multithreading
  static protected Executor executor;
  static public void setExecutor(Executor exec) {
    executor = exec;
  }

  static public void setTypicalDatasetMode(String mode) {
    if (mode.equalsIgnoreCase("random"))
      typicalDatasetMode = TypicalDataset.RANDOM;
    else if (mode.equalsIgnoreCase("latest"))
      typicalDatasetMode = TypicalDataset.LATEST;
    else if (mode.equalsIgnoreCase("penultimate"))
       typicalDatasetMode = TypicalDataset.PENULTIMATE;
    else if (mode.equalsIgnoreCase("first"))
       typicalDatasetMode = TypicalDataset.FIRST;
     else
      logger.error("Unknown setTypicalDatasetMode= " + mode);
  }

  static protected boolean debug = false, debugOpenFile = false, debugSyncDetail = false, debugProxy = false,
      debugRead = false, debugDateParse = false, debugConvert = false;

  //////////////////////////////////////////////////////////////////////////////////////////

  protected NetcdfDataset ncDataset; // the aggregation belongs to this dataset
  protected Type type; // the aggregation type
  protected Object spiObject; // pass to NetcdfFile.open()

  protected List<Aggregation.Dataset> explicitDatasets = new ArrayList<>(); // explicitly created Dataset objects from netcdf elements
  protected List<Aggregation.Dataset> datasets = new ArrayList<>(); // all : explicit and scanned
  protected MFileCollectionManager datasetManager; // manages scanning
  protected boolean cacheDirty = true; // aggCache persist file needs updating

  protected String dimName; // the aggregation dimension name

  private Element mergeNcml = null;

  // experimental
  protected String dateFormatMark;
  //protected EnumSet<NetcdfDataset.Enhance> enhance = null; // default no enhancement
  protected boolean isDate = false;
  protected DateFormatter dateFormatter = new DateFormatter();

  /**
   * Create an Aggregation for the given NetcdfDataset.
   * The following addXXXX methods are called, then finish(), before the object is ready for use.
   *
   * @param ncd      Aggregation belongs to this NetcdfDataset
   * @param dimName  the aggregation dimension name
   * @param type     the Aggregation.Type
   * @param recheckS how often to check if files have changes
   */
  protected Aggregation(NetcdfDataset ncd, String dimName, Type type, String recheckS) {
    this.ncDataset = ncd;
    this.dimName = dimName;
    this.type = type;
    String name = ncd.getLocation();
    if (name == null) name = "Agg-"+Integer.toString(ncd.hashCode());
    datasetManager = MFileCollectionManager.openWithRecheck(name, recheckS);
  }

  /**
   * Add a nested dataset, specified by an explicit netcdf element.
   * enhance is handled by the reader, so its always false here.
   *
   * @param cacheName   a unique name to use for caching
   * @param location    attribute "location" on the netcdf element
   * @param id          attribute "id" on the netcdf element
   * @param ncoordS     attribute "ncoords" on the netcdf element
   * @param coordValueS attribute "coordValue" on the netcdf element
   * @param sectionSpec attribute "section" on the netcdf element
   * @param reader      factory for reading this netcdf dataset
   */
  public void addExplicitDataset(String cacheName, String location, String id, String ncoordS, String coordValueS, String sectionSpec,
                                 ucar.nc2.util.cache.FileFactory reader) {

    Dataset nested = makeDataset(cacheName, location, id, ncoordS, coordValueS, sectionSpec, null, reader);
    explicitDatasets.add(nested);
  }

  public void addDataset(Dataset nested) {
    explicitDatasets.add(nested);
  }

  /**
   * Add a dataset scan
   *
   * @param crawlableDatasetElement defines a CrawlableDataset, or null
   * @param dirName             scan this directory
   * @param suffix              filter on this suffix (may be null)
   * @param regexpPatternString include if full name matches this regular expression (may be null)
   * @param dateFormatMark      create dates from the filename (may be null)
   * @param enhanceMode         how should files be enhanced
   * @param subdirs             equals "false" if should not descend into subdirectories
   * @param olderThan           files must be older than this time (now - lastModified >= olderThan); must be a time unit, may ne bull
   */
  public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix,
          String regexpPatternString, String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan) {

    datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, enhanceMode);

    this.dateFormatMark = dateFormatMark;
    if (dateFormatMark != null) {
      isDate = true;
      if (type == Type.joinExisting) type = Type.joinExistingOne; // tricky
      DateExtractor dateExtractor = new DateExtractorFromName(dateFormatMark, true);
      datasetManager.setDateExtractor(dateExtractor);
    }
 }

  // experimental
  public void addCollection(String spec, String olderThan) throws IOException {
    datasetManager = MFileCollectionManager.open(spec, spec, olderThan, new Formatter());
 }

  public void setModifications(Element ncmlMods) {
    this.mergeNcml = ncmlMods;
  }

  /**
   * Get type of aggregation
   *
   * @return type of aggregation
   */
  public Type getType() {
    return type;
  }

  /**
   * Get dimension name to join on
   *
   * @return dimension name or null if type union/tiled
   */
  public String getDimensionName() {
    return dimName;
  }

  protected String getLocation() {
    return ncDataset.getLocation();
  }

  /////////////////////////////////////////////////////////////////////

  public void close() throws IOException {
    persistWrite();
  }

  /**
   * Check to see if its time to rescan directory, and if so, rescan and extend dataset if needed.
   * Note that this just calls sync(), so structural metadata may be modified (!!)
   *
   * @return true if directory was rescanned and dataset may have been updated
   * @throws IOException on io error
   */
  public synchronized boolean syncExtend() throws IOException {
    return datasetManager.isScanNeeded() && _sync();
  }

  //public synchronized boolean sync() throws IOException {
  //  return datasetManager.isScanNeeded() && _sync();
  //}

  // LOOK could also use syncExtend()
  public long getLastModified() {
    try {
      datasetManager.scanIfNeeded();
    } catch (IOException e) {
      logger.error("Aggregation scan failed, e");
    }
    return datasetManager.getLastChanged();
  }

  private boolean _sync() throws IOException {
    if (!datasetManager.scan(true))
      return false; // nothing changed LOOK what about grib extention ??
    cacheDirty = true;
    makeDatasets(null);

    // rebuild the metadata
    rebuildDataset();
    ncDataset.finish();
    if (ncDataset.getEnhanceMode().contains(NetcdfDataset.Enhance.CoordSystems)) { // force recreation of the coordinate systems
      ncDataset.clearCoordinateSystems();
      ncDataset.enhance(ncDataset.getEnhanceMode());
      ncDataset.finish();
  }

    return true;
  }

  public String getFileTypeId() { // LOOK - should cache ??
     Dataset ds = null;
     NetcdfFile ncfile = null;
     try {
       ds = getTypicalDataset();
       ncfile = ds.acquireFile(null);
       return ncfile.getFileTypeId();

     } catch (Exception e) {
       logger.error("failed to open "+ds);

     } finally {
       if (ds != null) try {
         ds.close(ncfile);
       } catch (IOException e) {
         logger.error("failed to close "+ds);
  }
     }
     return "N/A";
   }

   public String getFileTypeDescription() { // LOOK - should cache ??
     Dataset ds = null;
     NetcdfFile ncfile = null;
     try {
       ds = getTypicalDataset();
       ncfile = ds.acquireFile(null);
       return ncfile.getFileTypeDescription();

     } catch (Exception e) {
       logger.error("failed to open "+ds);

     } finally {
       if (ds != null) try {
         ds.close(ncfile);
       } catch (IOException e) {
         logger.error("failed to close "+ds);
       }
     }
     return "N/A";
   }

  ///////////////////////////////////////////////////////////////////////////////////////////////////////
  // stuff for subclasses to override

  /**
    * Call this to build the dataset objects in the NetcdfDataset
    *
    * @param cancelTask maybe cancel
    * @throws IOException on read error
    */
   protected abstract void buildNetcdfDataset(CancelTask cancelTask) throws IOException;

   /**
    * Call this when rescan has found changed datasets
    *
    * @throws IOException on read error
    */
   protected abstract void rebuildDataset() throws IOException;

  /**
   * Allow information to be make persistent. Overridden in AggregationExisting
   *
   * @throws IOException on error
   */
  public void persistWrite() throws IOException {
  }

  /**
   * read info from the persistent XML file, if it exists; overridden in AggregationExisting
   */
  protected void persistRead() {
  }

  public void getDetailInfo(Formatter f) {
    f.format("  Type=%s%n", type);
    f.format("  dimName=%s%n", dimName);
    f.format("  Datasets (%d) %n", datasets.size());
    for (Dataset ds : datasets)
      ds.show(f);
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////////////

  // all elements are processed, finish construction

  public void finish(CancelTask cancelTask) throws IOException {
    datasetManager.scan(true); // Make the list of Datasets, by scanning if needed.
    cacheDirty = true;
    makeDatasets(cancelTask);

    //ucar.unidata.io.RandomAccessFile.setDebugAccess( true);
    buildNetcdfDataset(cancelTask);
    //ucar.unidata.io.RandomAccessFile.setDebugAccess( false);
  }

  public List<Dataset> getDatasets() {
    return datasets;
  }

  /**
   * Make the list of Datasets, from explicit and scans.
   *
   * @param cancelTask user can cancel
   * @throws IOException on i/o error
   */
  protected void makeDatasets(CancelTask cancelTask) throws IOException {

    // heres where the results will go
    datasets = new ArrayList<>();

    for (MFile cd : datasetManager.getFilesSorted()) {
      datasets.add( makeDataset(cd));
    }

    // sort using Aggregation.Dataset as Comparator.
    // Sort by date if it exists, else filename.
    Collections.sort(datasets);

      /* optionally extract the date
      String dateCoordS = null;
      if (null != dateFormatMark) {
        String filename = myf.getName(); // LOOK operates on name, not path
        Date dateCoord = DateFromString.getDateUsingDemarkatedCount(filename, dateFormatMark, '#');
        dateCoordS = formatter.toDateTimeStringISO(dateCoord);
        if (debugDateParse) System.out.println("  adding " + myf.getPath() + " date= " + dateCoordS);
      } else {
        if (debugDateParse) System.out.println("  adding " + myf.getPath());
      }

      String location = myf.getPath();
      Aggregation.Dataset ds = makeDataset(location, location, null, null, dateCoordS, null, enhance, null);
      datasets.add(ds);
    }

    // Sort by date if it exists, else filename.
    Collections.sort(datasets, new Comparator<Aggregation.Dataset>() {
      public int compare(Aggregation.Dataset ds1, Aggregation.Dataset ds2) {
        if(ds1.cd == null)
           return ds1.getLocation().compareTo(ds2.getLocation()) ;
        if (ds1.cd.dateCoord != null) // LOOK can we generalize
          return ds1.cd.dateCoord.compareTo(ds2.cd.dateCoord);
        else
          return ds1.cd.file.getName().compareTo(ds2.cd.file.getName());
      }
    });  */

    // add the explicit datasets - these need to be kept in order
    // LOOK - should they be before or after scanned? Does it make sense to mix scan and explicit?
    // AggFmrcSingle sets explicit datasets - the scan is empty
    for (Aggregation.Dataset dataset : explicitDatasets) {
      datasets.add(dataset);
    }

    // Remove unreadable files (i.e. due to permissions) from the aggregation.
    // LOOK: Is this logic we should install "upstream", perhaps in MFileCollectionManager?
    // It would affect other collections than just NcML aggregation in that case.
    for (Iterator<Dataset> datasetsIter = datasets.iterator(); datasetsIter.hasNext(); ) {
      Dataset dataset = datasetsIter.next();

      Path datasetPath;
      if (dataset.getMFile() instanceof MFileOS) {
        datasetPath = ((MFileOS) dataset.getMFile()).getFile().toPath();
      } else if (dataset.getMFile() instanceof MFileOS7) {
        datasetPath = ((MFileOS7) dataset.getMFile()).getNioPath();
      } else {
        continue;
      }

      if (!Files.isReadable(datasetPath)) {  // File.canRead() is broken on Windows, but the JDK7 methods work.
        logger.warn("Aggregation member isn't readable (permissions issue?). Skipping: " + datasetPath);
        datasetsIter.remove();
      }
    }

    // check for duplicate location
    Set<String> dset = new HashSet<>( 2 * datasets.size());
    for (Aggregation.Dataset dataset : datasets) {
      if (dset.contains(dataset.cacheLocation))
        logger.warn("Duplicate dataset in aggregation = "+dataset.cacheLocation);
      dset.add(dataset.cacheLocation);
    }

    if (datasets.size() == 0) {
      throw new IllegalStateException("There are no datasets in the aggregation " + datasetManager);
    }
  }

  /**
   * Open one of the nested datasets as a template for the aggregation dataset.
   *
   * @return a typical Dataset
   * @throws IOException if there are no datasets
   */
  protected Dataset getTypicalDataset() throws IOException {
    List<Dataset> nestedDatasets = getDatasets();
    int n = nestedDatasets.size();
    if (n == 0)
      throw new FileNotFoundException("No datasets in this aggregation");

    int select;
    if (typicalDatasetMode == TypicalDataset.LATEST)
      select = n - 1;
    else if (typicalDatasetMode == TypicalDataset.PENULTIMATE)
      select = (n < 2) ? 0 : n - 2;
    else if (typicalDatasetMode == TypicalDataset.FIRST)
      select = 0;
    else { // random is default
      if (r == null) r = new Random();
      select = (n < 2) ? 0 : r.nextInt(n);
    }

    return nestedDatasets.get(select);
  }
  private Random r;

  //////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Read an aggregation variable: A variable whose data spans multiple files.
   *
   * @param mainv      the aggregation variable
   * @param cancelTask allow the user to cancel
   * @return the data array
   * @throws IOException
   */
  //public abstract Array read(Variable mainv, CancelTask cancelTask) throws IOException;
  //public abstract Array reallyRead() throws IOException;

  /**
   * Read a section of an aggregation variable.
   *
   * @param mainv      the aggregation variable
   * @param cancelTask allow the user to cancel
   * @param section    read just this section of the data, refers to aggregated Variable's section.
   * @return the data array section
   * @throws IOException
   */
  //public abstract Array read(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException;
  //public abstract Array reallyRead(Section section) throws IOException, InvalidRangeException;

  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Dataset factory, so subclasses can override
   *
   * @param cacheName   a unique name to use for caching
   * @param location    attribute "location" on the netcdf element
   * @param id          attribute "id" on the netcdf element
   * @param ncoordS     attribute "ncoords" on the netcdf element
   * @param coordValueS attribute "coordValue" on the netcdf element
   * @param sectionSpec attribute "sectionSpec" on the netcdf element
   * @param enhance     open dataset in enhance mode NOT USED
   * @param reader      factory for reading this netcdf dataset
   * @return a Aggregation.Dataset
   */
  protected Dataset makeDataset(String cacheName, String location, String id, String ncoordS, String coordValueS,
          String sectionSpec, EnumSet<NetcdfDataset.Enhance> enhance, ucar.nc2.util.cache.FileFactory reader) {
    return new Dataset(cacheName, location, id, enhance, reader); // overridden in OuterDim, tiled
  }

  protected Dataset makeDataset(MFile dset) {
    return new Dataset(dset);
  }

  /**
   * Encapsolates a NetcdfFile that is a component of the aggregation.
   */
  public class Dataset implements Comparable {
    MFile mfile;
    protected String id; // id attribute on the netcdf element

    // deferred opening
    protected String cacheLocation;
    protected ucar.nc2.util.cache.FileFactory reader;
    protected Set<NetcdfDataset.Enhance> enhance; // used by Fmrc to read enhanced datasets
    protected DatasetUrl durl = null;

    /*
     * For subclasses.
     *
     * @param location location attribute on the netcdf element
     *
    protected Dataset(String location) {
      this.location = (location == null) ? null : StringUtil2.substitute(location, "\\", "/");
    }  */

    protected Dataset(MFile mfile) {
      this.mfile = mfile;
      this.cacheLocation = mfile.getPath();
      this.enhance = (Set<NetcdfDataset.Enhance>) mfile.getAuxInfo();
    }

    /**
     * Dataset constructor.
     * With this constructor, the actual opening of the dataset is deferred, and done by the reader.
     * Used with explicit netcdf elements, and scanned files.
     *
     * @param cacheLocation a unique name to use for caching
     * @param location  attribute "location" on the netcdf element
     * @param id  attribute "id" on the netcdf element
     * @param enhance   open dataset in enhance mode, may be null NOT USED
     * @param reader    factory for reading this netcdf dataset; if null, use NetcdfDataset.open( location)
     */
    protected Dataset(String cacheLocation, String location, String id, EnumSet<NetcdfDataset.Enhance> enhance, ucar.nc2.util.cache.FileFactory reader) {
      this.mfile = MFileOS.getExistingFile(location);
      this.cacheLocation = cacheLocation;
      this.id = id;
      //this.enhance = enhance;  // LOOK why ??
      this.reader = reader;
    }

    /**
     * Get the location of this Dataset
     *
     * @return the location of this Dataset
     */
    public String getLocation() {
      return (mfile == null) ? cacheLocation : mfile.getPath();
    }

    /**
     *
     * @return MFile or null
     */
    public MFile getMFile() {
      return mfile;
    }

    public String getCacheLocation() {
      return cacheLocation;
    }

    public String getId() {
      if (id != null) return id;
      if (mfile != null) return mfile.getPath();
      return Integer.toString(this.hashCode());
    }

    public NetcdfFile acquireFile(CancelTask cancelTask) throws IOException {
      if (debugOpenFile) System.out.println(" try to acquire " + cacheLocation);
      long start = System.currentTimeMillis();

      if (durl == null)
        durl = DatasetUrl.findDatasetUrl(cacheLocation); // cache the ServiceType so we dont have to keep figuring it out
      NetcdfFile ncfile = NetcdfDataset.acquireFile(reader, null, durl, -1, cancelTask, spiObject);

      // must merge NcML before enhancing
      if (mergeNcml != null)
        ncfile = NcMLReader.mergeNcML(ncfile, mergeNcml); // create new dataset
      if (enhance == null || enhance.isEmpty()) {
        if (debugOpenFile) System.out.println(" acquire (no enhance) " + cacheLocation + " took " + (System.currentTimeMillis() - start));
        return ncfile;
      }

      // must enhance
      NetcdfDataset ds;
      if (ncfile instanceof NetcdfDataset) {
        ds = (NetcdfDataset) ncfile;
        ds.enhance(enhance); // enhance "in place", ie modify the NetcdfDataset
      } else {
        ds = new NetcdfDataset(ncfile, enhance); // enhance when wrapping
      }

      if (debugOpenFile) System.out.println(" acquire (enhance) " + cacheLocation + " took " + (System.currentTimeMillis() - start));
      return ds;
    }

    protected void close(NetcdfFile ncfile) throws IOException {
      if (ncfile == null) return;
      cacheVariables(ncfile);
      ncfile.close();
    }

        // overridden in DatasetOuterDimension
    protected void cacheVariables(NetcdfFile ncfile) throws IOException {
    }

    public void show(Formatter f) {
      f.format("   %s%n", mfile.getPath());
    }

    protected Array read(Variable mainv, CancelTask cancelTask) throws IOException {
      NetcdfFile ncd = null;
      try {
        ncd = acquireFile(cancelTask);
        if ((cancelTask != null) && cancelTask.isCancel())
          return null;

        Variable v = findVariable(ncd, mainv);
        if (debugRead)
          System.out.printf("Agg.read %s from %s in %s%n", mainv.getNameAndDimensions(), v.getNameAndDimensions(), getLocation());
        return v.read();

      } finally {
        close( ncd);
      }
    }

    /**
     * Read a section of the local Variable.
     *
     * @param mainv      aggregated Variable
     * @param cancelTask let user cancel
     * @param section    reletive to the local Variable
     * @return the complete Array for mainv
     * @throws IOException           on I/O error
     * @throws InvalidRangeException on section error
     */
    protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section) throws  IOException, InvalidRangeException {
      NetcdfFile ncd = null;
      try {
        ncd = acquireFile(cancelTask);
        if ((cancelTask != null) && cancelTask.isCancel())
          return null;

        Variable v = findVariable(ncd, mainv);
        if (debugRead) {
          Section want = new Section(section);
          System.out.printf("Agg.read(%s) %s from %s in %s%n", want, mainv.getNameAndDimensions(), v.getNameAndDimensions(), getLocation());
        }

        return v.read(section);

      } finally {
        close( ncd);
      }
    }

    protected Variable findVariable(NetcdfFile ncfile, Variable mainV) {
      Variable v = ncfile.findVariable(mainV.getFullNameEscaped());
      if (v == null) {  // might be renamed
        VariableEnhanced ve = (VariableEnhanced) mainV;
        v = ncfile.findVariable(ve.getOriginalName()); // LOOK not escaped
      }
      return v;
    }

    // Datasets with the same locations are equal
    public boolean equals(Object oo) {
      if (this == oo) return true;
      if (!(oo instanceof Dataset)) return false;
      Dataset other = (Dataset) oo;
      return getLocation().equals(other.getLocation());
    }

    public int hashCode() {
      return getLocation().hashCode();
    }

    @Override
    public int compareTo(Object o) {
      Dataset other = (Dataset) o;
      return getLocation().compareTo( other.getLocation());
    }
  } // class Dataset

  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * All non-agg variables use a proxy to acquire the file before reading.
   * If the variable is caching, read data into cache now.
   * If not caching, VariableEnhanced.setProxyReader() is called.
   *
   * @param typicalDataset read from a "typical dataset"
   * @param newds          containing dataset
   * @throws IOException on i/o error
   */
  protected void setDatasetAcquireProxy(Dataset typicalDataset, NetcdfDataset newds) throws IOException {
    DatasetProxyReader proxy = new DatasetProxyReader(typicalDataset);
    setDatasetAcquireProxy( proxy, newds.getRootGroup());
  }

  protected void setDatasetAcquireProxy(DatasetProxyReader proxy, Group g) throws IOException {

    // all normal (non agg) variables must use a proxy to lock the file
    for (Variable v : g.getVariables()) {

      if (v.getProxyReader() != v) {
        if (debugProxy) System.out.println(" debugProxy: hasProxyReader " + v.getFullName());
        continue; // dont mess with agg variables
      }

      if (v.isCaching()) {  // cache the small ones
          v.setCachedData( v.read()); // cache the variableDS directly

      } else { // put proxy on the rest
        v.setProxyReader(proxy);
        if (debugProxy) System.out.println(" debugProxy: set proxy on " + v.getFullName());
      }
    }

    // recurse
    for (Group nested : g.getGroups()) {
      setDatasetAcquireProxy(proxy, nested);
    }
  }

  protected class DatasetProxyReader implements ProxyReader {
    Dataset dataset;

    DatasetProxyReader(Dataset dataset) {
      this.dataset = dataset;
    }

    @Override
    public Array reallyRead(Variable mainV, CancelTask cancelTask) throws IOException {
      NetcdfFile ncfile = null;
      try {
        ncfile = dataset.acquireFile(cancelTask);
        if ((cancelTask != null) && cancelTask.isCancel()) return null;
        Variable proxyV = findVariable(ncfile, mainV);
        return proxyV.read();
      } finally {
        dataset.close( ncfile);
      }
    }

    @Override
    public Array reallyRead(Variable mainV, Section section, CancelTask cancelTask) throws  IOException, InvalidRangeException {
      NetcdfFile ncfile = null;
      try {
        ncfile = dataset.acquireFile(cancelTask);
        Variable proxyV = findVariable(ncfile, mainV);
        if ((cancelTask != null) && cancelTask.isCancel()) return null;
        return proxyV.read(section);

      } finally {
        dataset.close( ncfile);
      }
    }
  }
  protected Variable findVariable(NetcdfFile ncfile, Variable mainV) {
    Variable v = ncfile.findVariable(mainV.getFullNameEscaped());
    if (v == null) {  // might be renamed
      VariableEnhanced ve = (VariableEnhanced) mainV;
      v = ncfile.findVariable(ve.getOriginalName()); // LOOK not escaped
    }
    return v;
  }

}


/* Copyright */
package thredds.server.catalog.tracker;

import org.jdom2.Element;
import org.jdom2.output.XMLOutputter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.client.catalog.Dataset;
import thredds.server.catalog.DatasetScan;
import thredds.server.catalog.FeatureCollectionRef;

import java.util.Formatter;
import java.util.HashMap;
import java.util.Map;

/**
 * DatasetTracker using in memory hash table
 *
 * @author caron
 * @since 6/6/2015
 */
public class DatasetTrackerInMem implements DatasetTracker {
  static private final Logger logger = LoggerFactory.getLogger(DatasetTracker.class);
  //static final boolean debugResourceControl = false;

  // resource control
  private HashMap<String, String> resourceControlHash = new HashMap<>(); // path, restrictAccess string for datasets
  // private volatile PathMatcher<String> resourceControlMatcher = new PathMatcher<>(); // path, restrictAccess string for datasetScan
  private boolean hasResourceControl = false;

  // InvDataset (not DatasetScan, DatasetFmrc) that have an NcML element in it. key is the request Path
  private Map<String, String> ncmlDatasetHash = new HashMap<>();
  //private int count;

  public boolean init(String dirPath, long maxDatasets) {
    return true;
  }

  public boolean trackDataset(long catId, Dataset dataset, Callback callback) {
    if (callback != null) callback.hasDataset(dataset);

    if (dataset.getRestrictAccess() != null) {
      if (callback != null) callback.hasRestriction(dataset);
      else putResourceControl(dataset);
    }

    // dont track ncml for DatasetScan or FeatureCollectionRef
    if (dataset instanceof DatasetScan) return false;
    if (dataset instanceof FeatureCollectionRef) return false;

    if (dataset.getNcmlElement() != null) {
      if (callback != null) callback.hasNcml(dataset);
      // want the string representation
      Element ncmlElem = dataset.getNcmlElement();
      XMLOutputter xmlOut = new XMLOutputter();
      String ncml = xmlOut.outputString(ncmlElem);
      System.out.printf("%s%n", ncml);
      ncmlDatasetHash.put(dataset.getUrlPath(), ncml);
    }
    return true;
  }

  /**
   * This tracks Dataset elements that have resource control attributes
   *
   * @param ds the dataset
   */
  void putResourceControl(Dataset ds) {
    if (logger.isDebugEnabled()) logger.debug("putResourceControl " + ds.getRestrictAccess() + " for " + ds.getName());
    resourceControlHash.put(ds.getUrlPath(), ds.getRestrictAccess());

    // resourceControl is inherited, but no guarentee that children paths are related, unless its a
    //   DatasetScan or InvDatasetFmrc. So we keep track of all datasets that have a ResourceControl, including children
    // DatasetScan and InvDatasetFmrc must use a PathMatcher, others can use exact match (hash)

    /* if (ds instanceof DatasetScan) {
      DatasetScan scan = (DatasetScan) ds;
      if (debugResourceControl)
        System.out.println("putResourceControl " + ds.getRestrictAccess() + " for datasetScan " + scan.getPath());
      resourceControlMatcher.put(scan.getPath(), ds.getRestrictAccess());

    } else { // dataset
      if (debugResourceControl)
        System.out.println("putResourceControl " + ds.getRestrictAccess() + " for dataset " + ds.getUrlPath());

      // LOOK: seems like you only need to add if InvAccess.InvService.isReletive
      // LOOK: seems like we should use resourceControlMatcher to make sure we match .dods, etc
      for (Access access : ds.getAccess()) {
        if (access.getService().isRelativeBase())
          resourceControlHash.put(access.getUrlPath(), ds.getRestrictAccess());
      }
    }  */

    hasResourceControl = true;
  }

  public String findResourceControl(String path) {
    if (!hasResourceControl) return null;

    if (path.startsWith("/"))
      path = path.substring(1);

    return resourceControlHash.get(path);
    //if (null == rc)
    //  rc = resourceControlMatcher.match(path);
    // return rc;
  }

  @Override
  public String findNcml(String path) {
    return ncmlDatasetHash.get(path);
  }

  @Override
  public boolean exists() {
    return false;
  }

  @Override
  public boolean reinit() {
    resourceControlHash = new HashMap<>(); // path, restrictAccess string for datasets
    ncmlDatasetHash = new HashMap<>();
    return true;
  }

  @Override
  public void close() {
  }

  @Override
  public void save() {
  }

  @Override
  public void showDB(Formatter f) {
    f.format("DatasetTrackerInMem%n");
    for (Map.Entry<String, String> entry : ncmlDatasetHash.entrySet()) {
      f.format(" %s == %s%n", entry.getKey(), entry.getValue());
    }
  }
}


package ucar.nc2.ft2.simpgeometry;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import ucar.ma2.Array;
import ucar.ma2.IndexIterator;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.Variable;
import ucar.nc2.constants.CF;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.ft2.simpgeometry.exception.InvalidDataseriesException;

/**
 * A CF 1.8 compliant Line
 * for use with Simple Geometries. Can also
 * represent Multilines.
 *
 * @author wchen@usgs.gov
 *
 */
public class CFLine implements Line {

	private static final Logger cfl = LoggerFactory.getLogger(CFLine.class);
	private List<Point> points;	// a list of the constitutent points of the Line, connected in ascending order as in the CF convention
	private Line next;	// if non-null, next refers to the next line part of a multi-line
	private Line prev;	// if non-null, prev refers to the previous line part of a multi-line
	private Array data;		// data associated with the line

	/**
	 * Get the geometry the data is associated with.
	 *
	 */
	public GeometryType getGeometryType() {
		return GeometryType.LINE;
	}

	/**
	 * Add a point to the end of the line.
	 *
	 */
	public void addPoint(double x, double y) {
		Point ptPrev = null;

		if(points.size() > 0) {
			ptPrev = points.get(points.size() - 1);
		}

		this.points.add(new CFPoint(x, y, ptPrev, null, null));

	}

	/**
	 * Returns the list of points which make up this line
	 *
	 * @return points - the collection of points that make up this line
	 */
	public List<Point> getPoints() {
		return points;
	}

	/**
	 * Get the data associated with this line
	 *
	 * @return data
	 */
	public Array getData() {
		return data;
	}

	/**
	 * If part of a multiline, returns the next line within that line
	 * if it is present.
	 *
	 * @return next line if present, null if not
	 */
	public Line getNext() {
		return next;
	}

	/**
	 * If part of a multiline, returns the previous line within that line
	 * if it is present
	 *
	 * @return previous line if present, null if not
	 */
	public Line getPrev() {
		return prev;
	}

	/**
	 * Set the data associated with this Line
	 *
	 * @param data - array of data to set to
	 */
	public void setData(Array data) {
		this.data = data;
	}

	/**
	 * Sets the previous line which makes up the multiline which this line is a part of.
	 * If prev is a CFLine, automatically connect the other line to this line as well.
	 *
	 */
	public void setNext(Line next) {
		if(next instanceof CFLine) {
			setNext((CFLine) next);
		}

		else this.next = next;
	}

	/**
	 * Sets the next line which makes up the multiline which this line is a part of.
	 * Automatically connects the other line to this line as well.
	 */
	protected void setNext(CFLine next) {
		this.next = next;

		if(next != null) {
			next.setPrevOnce(this);
		}
	}

	private void setNextOnce(CFLine next) {
		this.next = next;
	}

	/**
	 * Sets the previous line which makes up the multiline which this line is a part of.
	 * If prev is a CFLine, automatically connect the other line to this line as well.
	 */
	public void setPrev(Line prev) {
		if(prev instanceof CFLine)
		{
			setPrev((CFLine) prev);
		}

		else this.prev = prev;
	}

	/**
	 * Sets the previous line which makes up the multiline which this line is a part of.
	 * Automatically connect the other line to this line as well.
	 */
	protected void setPrev(CFLine prev) {
		this.prev = prev;

		if(prev != null) {
			prev.setNextOnce(this);
		}
	}

	private void setPrevOnce(CFLine prev) {
		this.prev = prev;
	}

	/**
	 * Given a dataset, variable, and index, automatically populates this Line and
	 * returns it. If not found, returns null.
	 *
	 * @param dataset which the variable is a part of
	 * @param var the variable which has a geometry attribute
	 * @param index of the line within the variable
	 * @return return a line
	 */
	public Line setupLine(NetcdfDataset dataset, Variable var, int index)
	{
		this.points.clear();
		Array xPts = null;
		Array yPts = null;
		Variable nodeCounts = null;
		Variable partNodeCounts = null;

		List<CoordinateAxis> axes = dataset.getCoordinateAxes();
		CoordinateAxis x = null; CoordinateAxis y = null;

		String[] nodeCoords = var.findAttributeIgnoreCase(CF.NODE_COORDINATES).getStringValue().split(" ");

		// Look for x and y

		for(CoordinateAxis ax : axes){

			if(ax.getFullName().equals(nodeCoords[0])) x = ax;
			if(ax.getFullName().equals(nodeCoords[1])) y = ax;
		}

		// Affirm node counts
		String node_c_str = var.findAttValueIgnoreCase(CF.NODE_COUNT, "");

		if(!node_c_str.equals("")) {
			nodeCounts = dataset.findVariable(node_c_str);
		}

		else return null;

		// Affirm part node counts
		String pNodeCoStr = var.findAttValueIgnoreCase(CF.PART_NODE_COUNT, "");

		if(!pNodeCoStr.equals("")) {
			partNodeCounts = dataset.findVariable(pNodeCoStr);
		}

		SimpleGeometryIndexFinder indexFinder = new SimpleGeometryIndexFinder(nodeCounts);

		//Get beginning and ending indicies for this polygon
		int lower = indexFinder.getBeginning(index);
		int upper = indexFinder.getEnd(index);

		try {

			xPts = x.read( lower + ":" + upper ).reduce();
			yPts = y.read( lower + ":" + upper ).reduce();

			IndexIterator itrX = xPts.getIndexIterator();
			IndexIterator itrY = yPts.getIndexIterator();

			// No multipolygons just read in the whole thing
			if(partNodeCounts == null) {

				this.next = null;
				this.prev = null;

				// x and y should have the same shape, will add some handling on this
				while(itrX.hasNext()) {
					this.addPoint(itrX.getDoubleNext(), itrY.getDoubleNext());
				}

				switch(var.getRank()) {

				case 2:
					this.setData(var.read(CFSimpleGeometryHelper.getSubsetString(var, index)).reduce());
					break;

				case 1:
					this.setData(var.read("" + index));
					break;

				default:
					throw new InvalidDataseriesException(InvalidDataseriesException.RANK_MISMATCH);	// currently do not support anything but dataseries and scalar associations

				}
			}

			// If there are multipolygons then take the upper and lower of it and divy it up
			else {

				Line tail = this;
				Array pnc = partNodeCounts.read();
				IndexIterator pncItr = pnc.getIndexIterator();

				// In part node count search for the right index to begin looking for "part node counts"
				int pncInd = 0;
				int pncEnd = 0;
				while(pncEnd < lower)
				{
					pncEnd += pncItr.getIntNext();
					pncInd++;
				}

				// Now the index is found, use part node count and the index to find each part node count of each individual part
				while(lower < upper) {

					int smaller = pnc.getInt(pncInd);

					while(smaller > 0) {
						tail.addPoint(itrX.getDoubleNext(), itrY.getDoubleNext());
						smaller--;
					}

					// Set data of each
					switch(var.getRank()) {

					case 2:
						tail.setData(var.read(CFSimpleGeometryHelper.getSubsetString(var, index)).reduce());
						break;

					case 1:
						tail.setData(var.read("" + index));
						break;

					default:
						throw new InvalidDataseriesException(InvalidDataseriesException.RANK_MISMATCH);	// currently do not support anything but dataseries and scalar associations

					}

					lower += tail.getPoints().size();
					pncInd++;
					tail.setNext(new CFLine());
					tail = tail.getNext();
				}

				//Clean up
				tail = tail.getPrev();
				if(tail != null) tail.setNext(null);
			}
		}

		catch (IOException  | InvalidRangeException  | InvalidDataseriesException e) {
			cfl.error(e.getMessage());;
			return null;
		}

		return this;
	}

	/**
	 * Gets the upper bounding box coordinate on the line.
	 * @return double array = (x, y)
	 */
	public double[] getBBUpper() {
		double[] bbUpper = new double[2];

		List<Point> ptList = this.getPoints();
		if(ptList.isEmpty()) return null;
		bbUpper[0] = ptList.get(0).getY();
		bbUpper[1] = ptList.get(0).getY();

		for(Point pt : this.getPoints()) {
			if(bbUpper[0] < pt.getX()){
				bbUpper[0] = pt.getX();
			}

			if(bbUpper[1] < pt.getY()) {
				bbUpper[1] = pt.getY();
			}
		}

		// Got maximum points, add some padding.
		bbUpper[0] += 10;
		bbUpper[1] += 10;

		return bbUpper;
	}

	/**
	 * Gets the lower bounding box coordinate on the line.
	 * @return double array = (x, y)
	 */
	public double[] getBBLower() {
		double[] bbLower = new double[2];

		List<Point> ptList = this.getPoints();
		if(ptList.isEmpty()) return null;
		bbLower[0] = ptList.get(0).getY();
		bbLower[1] = ptList.get(0).getY();

		for(Point pt : this.getPoints()) {
			if(bbLower[0] > pt.getX()){
				bbLower[0] = pt.getX();
			}

			if(bbLower[1] > pt.getY()) {
				bbLower[1] = pt.getY();
			}
		}

		// Got minimum points, add some padding.
		bbLower[0] -= 10;
		bbLower[1] -= 10;

		return bbLower;
	}

	/**
	 *  Constructs an "empty" line with no members using an ArrayList to implement the point list.
	 *
	 */
	public CFLine() {
		this.points = new ArrayList<Point>();
		this.next = null;
		this.prev = null;
		this.data = null;
	}

	/**
	 * From a given list of points, construct a line
	 *
	 * @param newPt The list of points which will constitute the new line
	 */
	public CFLine(List<Point> newPt) {
		this.points = newPt;
		this.next = null;
		this.data = null;
	}

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.cataloggen.config;

import thredds.catalog.InvDatasetImpl;
import thredds.catalog.InvDataset;

import java.util.List;
import java.util.Iterator;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 * Provides filtering capabilities for InvDatasets.
 *

 * The static method acceptDatasetByFilterGroup() is provided to allow
 * filtering with a group of filters. To make filtering out a set of
 * datasets as easy as allowing a set of datasets, DatasetFilter provides
 * a dataset reject mode as well as a dataset accept mode. [Notes:
 * 1) rejection of a dataset takes precedence over acceptance; and 2) all
 * datasets are accepted if the filter group is empty.]
 *

 * A DatasetFilter instance contains information on how to filter datasets
 * and can be set to either accept or reject datasets that match
 * the filter criteria (accept is the default). The accept() method
 * should only be called if the isAcceptMatchingDatasets() method returns
 * true. Similarly, the reject() method should only be called if the
 * isRejectMatchingDatasets() method returns true.
 *
 * @author Ethan Davis
 * @since 2002-12-11T15:27+0700
 */

public class DatasetFilter {
  private DatasetSource parentDatasetSource = null;

  // attributes of the datasetFilter element
  private String name = null;
  private DatasetFilter.Type type = null;
  private String matchPattern = null;
  protected Pattern regExpPattern;

  /**
   * Specifies the target of the matchPattern.
   */
  private String matchPatternTarget = null;

  /**
   * If true this DatasetFilter applies to collection datasets, if false
   * it does not (the default value is false).
   */
  private boolean applyToCollectionDatasets = false;
  /**
   * If true this DatasetFilter applies to atomic datasets, if false it
   * does not (the default value is true).
   */
  private boolean applyToAtomicDatasets = true;

  /**
   * Indicates whether datasets that match the filter criteria are accepted
   * or rejected (accept is the default).
   */
  private boolean rejectMatchingDatasets = false;

  // validation flag and log
  private boolean isValid = true;
  private StringBuffer log = new StringBuffer();

  /**
   * Constructor
   *
   * @param parentDsSource
   * @param name
   * @param type
   * @param matchPattern
   */
  public DatasetFilter(DatasetSource parentDsSource, String name,
                       DatasetFilter.Type type,
                       String matchPattern) {
    // Check that given type is not null.
    if (type == null) {
      isValid = false;
      log.append(" ** DatasetFilter (1): invalid type for datasetFilter (" + name + ")");
    }
    this.parentDatasetSource = parentDsSource;
    this.name = name;
    this.type = type;

    if (matchPattern == null) {
      isValid = false;
      log.append(" ** DatasetFilter (2): null matchPattern not allowed.");
    } else {
      this.matchPattern = matchPattern;
      try {
        this.regExpPattern = java.util.regex.Pattern.compile(this.matchPattern);
      }
      catch (PatternSyntaxException e) {
        isValid = false;
        log.append(" ** DatasetFilter (3): invalid matchPattern [" + this.matchPattern + "].");
      }
    }
  }

  public DatasetFilter(DatasetSource parentDsSource, String name, DatasetFilter.Type type,
                       String matchPattern, boolean applyToCollectionDatasets,
                       boolean applyToAtomicDatasets, boolean rejectMatchingDatasets) {
    this(parentDsSource, name, type, matchPattern);
    this.applyToCollectionDatasets = applyToCollectionDatasets;
    this.applyToAtomicDatasets = applyToAtomicDatasets;
    this.rejectMatchingDatasets = rejectMatchingDatasets;
  }

  /**
   * Return the parent DatasetSource of this DatasetFilter
   */
  public DatasetSource getParentDatasetSource() {
    return (this.parentDatasetSource);
  }

  /**
   * Return the name of this DatasetFilter
   *
   * @return String the name of this.
   */
  public String getName() {
    return (this.name);
  }

  /**
   * Return the type of this DatasetFilter
   *
   * @return DatasetFilter.Type the type of this.
   */
  public DatasetFilter.Type getType() {
    return (this.type);
  }

  /**
   * Return the matchPattern of this DatasetFilter
   *
   * @return String the matchPattern of this.
   */
  public String getMatchPattern() {
    return (this.matchPattern);
  }

  public String getMatchPatternTarget() {
    return matchPatternTarget;
  }

  public void setMatchPatternTarget(String matchPatternTarget) {
    this.matchPatternTarget = matchPatternTarget;
  }

  public boolean isApplyToCollectionDatasets() {
    return (this.applyToCollectionDatasets);
  }

  public void setApplyToCollectionDatasets(boolean applyToCollectionDatasets) {
    this.applyToCollectionDatasets = applyToCollectionDatasets;
  }

  public boolean isApplyToAtomicDatasets() {
    return (this.applyToAtomicDatasets);
  }

  public void setApplyToAtomicDatasets(boolean applyToAtomicDatasets) {
    this.applyToAtomicDatasets = applyToAtomicDatasets;
  }

  public boolean isRejectMatchingDatasets() {
    return (this.rejectMatchingDatasets);
  }

  public boolean isAcceptMatchingDatasets() {
    return (!this.rejectMatchingDatasets);
  }

  public void setRejectMatchingDatasets(boolean rejectMatchingDatasets) {
    this.rejectMatchingDatasets = rejectMatchingDatasets;
  }

  /**
   * Validate this DatasetFilter object. Return true if valid, false if invalid.
   *
   * @param out StringBuffer with validation messages.
   * @return boolean true if valid, false if invalid
   */
  boolean validate(StringBuilder out) {
    this.isValid = true;

    // If log from construction has content, append to validation output msg.
    if (this.log.length() > 0) {
      out.append(this.log);
    }

    // Validity check: 'name' cannot be null. (Though, 'name'
    // can be an empty string.)
    if (this.getName() == null) {
      isValid = false;
      out.append(" ** DatasetFilter (4): null value for name is not valid.");
    }

    // Check that type is not null.
    if (this.getType() == null) {
      isValid = false;
      out.append(" ** DatasetFilter (5): null value for type is not valid (set with bad string?).");
    }

    // Validity check: 'matchPattern' must be null if 'type' value
    // is not 'RegExp'.
    if (this.type == DatasetFilter.Type.REGULAR_EXPRESSION &&
            this.matchPattern == null) {
      isValid = false;
      out.append(" ** DatasetFilter (6): null value for matchPattern not valid when type is 'RegExp'.");
    }
    if (this.type != DatasetFilter.Type.REGULAR_EXPRESSION &&
            this.type != null &&
            this.matchPattern != null) {
      isValid = false;
      out.append(" ** DatasetFilter (7): matchPattern value (" + this.matchPattern +
              ") must be null if type is not 'RegExp'.");
    }

    return (this.isValid);
  }

  /**
   * string representation
   */
  public String toString() {
    StringBuffer tmp = new StringBuffer();
    tmp.append("DatasetFilter[name:<" + this.getName() +
            "> type:<" + this.getType() +
            "> matchPattern:<" + this.getMatchPattern() + ">");

    return (tmp.toString());
  }

  /**
   * Test whether the given dataset should be excluded from the dataset collection.
   *
   * @param dataset - the dataset to be tested
   * @return true if and only if the dataset should be excluded
   * @throws IllegalStateException if the filter accepts rather than rejects datasets.
   */
  public boolean reject(InvDataset dataset) {
    if (this.isAcceptMatchingDatasets())
      throw new IllegalStateException("Accept filter <" + this.getName() + "> does not allow call to reject().");
    return (this.match(dataset));
  }

  /**
   * Test whether the given dataset should be included in a dataset collection.
   *
   * @param dataset - the dataset to be tested
   * @return true if and only if the dataset should be included
   * @throws IllegalStateException if the filter rejects rather than accepts datasets.
   */
  public boolean accept(InvDataset dataset) {
    if (this.isRejectMatchingDatasets())
      throw new IllegalStateException("Reject filter <" + this.getName() + "> does not allow call to accept().");
    return (this.match(dataset));
  }

  protected boolean appliesToDataset(InvDataset dataset) {
    // Check whether this filter applies to the given dataset.
    if (this.getParentDatasetSource().isCollection(dataset)
            && !this.applyToCollectionDatasets)
      return (false);
    if ((!this.getParentDatasetSource().isCollection(dataset))
            && !this.applyToAtomicDatasets)
      return (false);
    return (true);
  }

  /**
   * Test whether the given dataset matches the filter criteria.
   *
   * @param dataset - the dataset to be tested.
   * @return true if and only if the dataset matches the filter criteria.
   */
  private boolean match(InvDataset dataset) {
    // Check whether this filter applies to the given dataset.
    if (this.getParentDatasetSource().isCollection(dataset) && !this.applyToCollectionDatasets)
      return (false);
    if ((!this.getParentDatasetSource().isCollection(dataset)) && !this.applyToAtomicDatasets)
      return (false);

    // Set the default matchPatternTarget so old versions still work.
    if (this.matchPatternTarget == null) {
      if (this.getParentDatasetSource().isCollection(dataset)) {
        this.setMatchPatternTarget("name");
      } else {
        this.setMatchPatternTarget("urlPath");
      }
    }

    if (this.type == DatasetFilter.Type.REGULAR_EXPRESSION) {
      boolean isMatch;
      if (this.getMatchPatternTarget().equals("name")) {
        java.util.regex.Matcher matcher = this.regExpPattern.matcher(dataset.getName());
        isMatch = matcher.find();
      } else if (this.getMatchPatternTarget().equals("urlPath")) {
        java.util.regex.Matcher matcher = this.regExpPattern.matcher(((InvDatasetImpl) dataset).getUrlPath());
        isMatch = matcher.find();
      } else {
        // ToDo deal with any matchPatternTarget (XPath-ish)
        isMatch = false;
      }

//      // Invert the meaning of a match (accept things that don't match).
//      if ( this.isRejectMatchingDatasets())
//      {
//        // If match, return false.
//        return( regExpMatch == null ? true : false );
//      }
//      // Don't invert (a match is a match).
//      else
//      {
      // If match, return true.
      return (isMatch);
//      }
    } else {
      System.err.println("WARNING -- DatasetFilter.accept(): unsupported type" +
              " <" + this.type.toString() + ">.");
      return (false);
      // @todo think about exceptions.
      //throw new java.lang.Exception( "DatasetFilter.accept():" +
      //  " unsupported type <" + this.type.toString() + ">.");
    }

  }

  /**
   * Given a dataset and a group of filters, return true if the group of
   * filters indicates that the dataset should be accepted, false if it
   * should be rejected.
   *

   * To make filtering out a set of datasets as easy as allowing a set of
   * datasets, DatasetFilter provides a dataset reject mode as well as a
   * dataset accept mode. Rejection of a dataset by any filter overrides
   * acceptance by any number of filters. Therefore, to be accepted by the
   * group a dataset needs to be accepted by at least one filter, however,
   * rejection by a single filter will cause rejection by the group.
   *

   * If the filter group is empty, the dataset will be accepted.
   *
   * @param filters             - the group of filters to apply to the dataset.
   * @param dataset             - the dataset on which to apply the filter group.
   * @param isCollectionDataset
   * @return true if the group of filters indicates that the dataset should be accepted, false otherwise.
   * @throws NullPointerException if the filter list or the dataset is null.
   */
  public static boolean acceptDatasetByFilterGroup(List filters, InvDataset dataset, boolean isCollectionDataset) {
    if (filters == null) throw new NullPointerException("Given null list of filters.");
    if (dataset == null) throw new NullPointerException("Given null dataset.");

    // If not filters, accept all datasets.
    if (filters.isEmpty())
      return (true);

    // Loop through DatasetFilter list to check if current dataset should be accepted.
    // @todo If none of the filters apply to directories, accept all directories.
    boolean accept = false;
    boolean anyApplyToAtomic = false;
    boolean anyApplyToCollection = false;
    for (Iterator it = filters.iterator(); it.hasNext();) {
      DatasetFilter curFilter = (DatasetFilter) it.next();
      anyApplyToAtomic |= curFilter.isApplyToAtomicDatasets();
      anyApplyToCollection |= curFilter.isApplyToCollectionDatasets();

      if (curFilter.isAcceptMatchingDatasets()) {
        if (curFilter.accept(dataset)) {
          accept = true; // Dataset accepted by current DatasetFilter.
        }
      } else // if ( ! curFilter.isAcceptMatchingDatasets())
      {
        if (curFilter.reject(dataset)) {
          return (false); // Rejection takes precedence over accpetance
        }
      }
    }

    // At least one filter accepted (and none rejected), so accept.
    if (accept) return (true);

    // Check if any filters apply to dataset. If none apply, accept the dataset.
    if (isCollectionDataset) {
      if (!anyApplyToCollection) return (true); // Collection ds, no collection filters.
    } else {
      if (!anyApplyToAtomic) return (true); // Atomic ds, no atomic filters.
    }

    // Dataset not accepted or rejected by any DatasetFilter (so reject).
    return (false);
  }

  enum Type {
    REGULAR_EXPRESSION("RegExp");

    private String altId;

    Type(String altId) {
      this.altId = altId;
    }

    public String toString() {
      return this.altId;
    }

    public static Type getType(String altId) {
      if (altId == null)
        return null;

      for (Type curType : Type.values()) {
        if (curType.altId.equals(altId))
          return curType;
      }
      return null;
    }
  }
}


package ucar.nc2.ft.point;

import java.io.File;
import java.io.IOException;
import java.util.*;

import com.google.common.base.Preconditions;
import ucar.nc2.constants.FeatureType;
import ucar.nc2.ft.FeatureDatasetFactoryManager;
import ucar.nc2.ft.FeatureDatasetPoint;
import ucar.nc2.ft.NoFactoryFoundException;
import ucar.nc2.ft.PointFeatureIterator;
import ucar.nc2.units.DateUnit;

/**
 *
 * @author cwardgar
 * @since 2014/08/21
 */
// This class ought to be a PointFeatureCollection, by extending PointCollectionImpl.
// However, we do not have the timeUnit and altUnits that the constructor requires. Does it really need
// that info? Can't it calculate it from one of its features? That interface may need to be re-thought.
public class SortingStationPointFeatureCache {
    public static final Comparator<StationPointFeature> stationNameComparator = (pointFeat1, pointFeat2) ->
            pointFeat1.getStation().getName().compareTo(pointFeat2.getStation().getName());

    private final SortedMap<StationPointFeature, List<StationPointFeature>> inMemCache;

    private volatile StationFeatureCopyFactory stationFeatCopyFactory;

    public SortingStationPointFeatureCache() {
        this(stationNameComparator);
    }

    // We're going to init stationFeatCopyFactory using the first feat that's add()ed.
    public SortingStationPointFeatureCache(Comparator<StationPointFeature> comp) {
        this.inMemCache = new TreeMap<>(Preconditions.checkNotNull(comp, "comp == null"));
        // stationFeatCopyFactory remains null.
    }

    public SortingStationPointFeatureCache(
            Comparator<StationPointFeature> comp, StationPointFeature proto, DateUnit dateUnit) throws IOException {
        this.inMemCache = new TreeMap<>(Preconditions.checkNotNull(comp, "comp == null"));

        if (proto != null && dateUnit != null) {
            this.stationFeatCopyFactory = new StationFeatureCopyFactory(proto);
        }
    }

    public void add(StationPointFeature feat) throws IOException {
        Preconditions.checkNotNull(feat, "feat == null");
        StationPointFeature featCopy = getStationFeatureCopyFactory(feat).deepCopy(feat);

        List<StationPointFeature> bucket = inMemCache.get(featCopy);
        if (bucket == null) {
            bucket = new LinkedList<>();
            inMemCache.put(featCopy, bucket);
        }

        bucket.add(featCopy);
    }

    public void addAll(File datasetFile) throws NoFactoryFoundException, IOException {
        try (FeatureDatasetPoint fdPoint = (FeatureDatasetPoint) FeatureDatasetFactoryManager.open(
                FeatureType.STATION, datasetFile.getAbsolutePath(), null, new Formatter())) {
            addAll(fdPoint);
        }
    }

    // fdPoint remains open.
    public void addAll(FeatureDatasetPoint fdPoint) throws IOException {
        try (PointFeatureIterator pointFeatIter =
                new FlattenedDatasetPointCollection(fdPoint).getPointFeatureIterator()) {
            while (pointFeatIter.hasNext()) {
                StationPointFeature pointFeat = (StationPointFeature) pointFeatIter.next();
                add(pointFeat);
            }
        }
    }

    // Double-check idiom for lazy initialization of instance fields. See Effective Java 2nd Ed, p. 283.
    private StationFeatureCopyFactory getStationFeatureCopyFactory(StationPointFeature proto) throws IOException {
        if (stationFeatCopyFactory == null) {
            synchronized (this) {
                if (stationFeatCopyFactory == null) {
                    stationFeatCopyFactory = createStationFeatureCopyFactory(proto);
                }
            }
        }

        assert stationFeatCopyFactory != null : "We screwed this up.";
        return stationFeatCopyFactory;
    }

    private StationFeatureCopyFactory createStationFeatureCopyFactory(StationPointFeature proto) throws IOException {
        return new StationFeatureCopyFactory(proto);
    }

    // TODO: Once this method is called, prohibit any further additions to cache.
    public PointFeatureIterator getPointFeatureIterator() throws IOException {
        return new PointIteratorAdapter(new Iter());
    }

    private class Iter implements Iterator<StationPointFeature> {
        private final Iterator<List<StationPointFeature>> bucketsIter;
        private Iterator<StationPointFeature> featsIter;

        public Iter() {
            this.bucketsIter = inMemCache.values().iterator();
        }

        @Override
        public boolean hasNext() {  // Method is idempotent.
            while (featsIter == null || !featsIter.hasNext()) {
                if (!bucketsIter.hasNext()) {
                    return false;
                } else {
                    featsIter = bucketsIter.next().iterator();
                }
            }

            assert featsIter != null && featsIter.hasNext();
            return true;
        }

        @Override
        public StationPointFeature next() {
            if (!hasNext()) {  // Don't rely on user to call this.
                throw new NoSuchElementException("There are no more elements.");
            } else {
                return featsIter.next();
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("Operation not supported by this iterator.");
        }
    }
}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.servers;

import java.util.*;
import java.io.*;

import opendap.dap.*;
import opendap.dap.parsers.*;
import opendap.servers.parsers.CeParser;
import opendap.servlet.ReqState;
import opendap.util.Debug;

/**
 * This class is used to parse and evaluate a constraint expression. When
 * constructed it must be passed a valid DDS along with the expression. This
 * DDS will be used as the environment (collection of variables and
 * functions) during the parse and evaluation of the constraint expression.
 *

 * A server (servlet, CGI, ...) must first instantiate the DDS (possibly
 * reading it from a cache) and then create and instance of this class. Once
 * created, the constraint may be parsed and then evaluated. The class
 * supports sending data based on the results of CE evaluation. That is, the
 * send() method of the class combines both the evaluation of the constraint
 * and the output of data values so that the server can return data using a
 * single method call.

 *

 * Custom parsing
 * The CEEvaluator parses constraint expressions into Clause objects
 * using a ClauseFactory. Customized behavior during parsing can be
 * achieved by passing a customized ClauseFactory into the CEEvaluator.

 *

 * Support for server side functions
 * Server side functions are supported via the FunctionLibrary class.
 * Custom server side function support is achieved by using
 * a customized ClauseFactory which in turn contains a customized
 * FunctionLibrary.

 *

 * More details are found in the documentation for the respective classes.
 *
 * @author jhrg
 * @author ndp
 * @author Joe Wielgosz (joew@cola.iges.org)
 * @version $Revision: 21071 $
 * @see ServerDDS
 * @see ServerMethods
 * @see ClauseFactory
 * @see FunctionLibrary
 * @see Clause
 */

public class CEEvaluator {

    private static boolean _Debug;

    /**
     * This contains the DDS to be used during parse and evaluation of the
     * CE.
     */
    private ServerDDS _dds;

    /**
     * The Clause objects which hold the parsed selection information.
     */
    private Vector _cv;

    /**
     * The factory which will be used by the parser to construct the clause
     * tree. This allows servers to pass in a factory which creates
     * custom clause objects.
     */
    private ClauseFactory clauseFactory;

    /**
     * Construct a new CEEvaluator with dds as the
     * DDS object with which to resolve all variable and function names.
     *
     * @param dds DDS object describing the dataset targeted by this
     *            constraint.
     */
    public CEEvaluator(ServerDDS dds) {
        _dds = dds;
        _cv = new Vector();
        _Debug = Debug.isSet("CE");
        if (_Debug)
            System.out.println("CE debugging enabled.");

    }

    /**
     * Construct a new CEEvaluator with dds as the
     * DDS object with which to resolve all variable and function names, and
     * clauseFactory as a source of Clause objects .
     *
     * @param clauseFactory The factory which will be used by the parser to construct the clause
     *                      tree. This allows servers to pass in a factory which creates
     *                      custom clause objects.
     * @param dds           DDS object describing the dataset targeted by this
     *                      constraint.
     */
    public CEEvaluator(ServerDDS dds, ClauseFactory clauseFactory) {
        _dds = dds;
        _cv = new Vector();
        _Debug = Debug.isSet("CE");
        if (_Debug)
            System.out.println("CE debugging enabled.");

        this.clauseFactory = clauseFactory;

        _Debug = Debug.isSet("CEEvaluator");
    }

    /**
     * Return a reference to the CEEvaluator's DDS object.
     */
    public ServerDDS getDDS() {
        return _dds;
    }

    /**
     * Parse a constraint expression. Variables in the projection are marked
     * as such in the CEEvaluator's ServerDDS instance. The selection
     * subexpression is then parsed and a list of Clause objects is built.
     *

     * The parser is located in opendap.servers.parsers.CeParser.
     *
     * @param constraint The constraint expression to parse.
     * @throws ParseException
     * @throws NoSuchVariableException
     * @throws NoSuchFunctionException
     * @throws InvalidOperatorException
     * @throws InvalidParameterException
     * @throws SBHException
     */
    public void parseConstraint(String constraint, String urlencoded)
            throws ParseException, opendap.dap.DAP2Exception, NoSuchVariableException,
            NoSuchFunctionException, InvalidOperatorException,
            InvalidParameterException, SBHException, WrongTypeException {

        if (clauseFactory == null) {
            clauseFactory = new ClauseFactory();
        }

        // Parses constraint expression (duh...) and sets the
        // projection flag for each member of the CE's ServerDDS
        // instance. This also builds the list of clauses.

	try {
        CeParser.constraint_expression(this,
            _dds.getFactory(),
            clauseFactory,
            constraint, urlencoded);
	} catch (ConstraintException ce) {
	    // convert to a DAP2Exception
        ce.printStackTrace();
	    throw new DAP2Exception(ce);
	}

        if (_Debug) {
            int it = 0;
            Enumeration ec = getClauses();

            System.out.println("Results of clause parsing:");
            if (!ec.hasMoreElements())
                System.out.println("    No Clauses Found.");

            while (ec.hasMoreElements()) {
                it++;
                System.out.println("    Clause " + it + ": " + ec.nextElement());
            }
        }

    }

    /**
     * Convenience wrapper for parseConstraint.
     *
     * @param rs
     * @throws ParseException
     * @throws opendap.dap.DAP2Exception
     * @throws NoSuchVariableException
     * @throws NoSuchFunctionException
     * @throws InvalidOperatorException
     * @throws InvalidParameterException
     * @throws SBHException
     * @throws WrongTypeException
     */
    public void parseConstraint(ReqState rs)
            throws ParseException, opendap.dap.DAP2Exception, NoSuchVariableException,
            NoSuchFunctionException, InvalidOperatorException,
            InvalidParameterException, SBHException, WrongTypeException
    {
           parseConstraint(rs.getConstraintExpression(),rs.getRequestURL().toString());
    }

    /**
     * Add a clause to the constraint expression.
     *
     * @param c The Clause to append.
     */
    public void appendClause(Clause c) {
        if (c != null) {
            _cv.add(c);
        }
    }

    /**
     * Remove a clause from the constraint expression. This will
     * will remove the first occurence of the passed clause from
     * the constraint expression. This is done be reference, so if
     * the passed Clause object is NOT already in the constraint
     * expression then nothing happens. And, if it should appear
     * more than once (which I don't think is possible) only
     * the first occurence will be removed.
     *
     * @param c The Clause to append.
     * @return True if constraint expression contained the passed Clause
     *         object and it was successfully removed.
     */
    public boolean removeClause(Clause c) {
        if (c != null) {
            return (_cv.remove(c));
        }
        return (false);
    }

    /**
     * Get access to the list of clauses built by parsing the selection part
     * of the constraint expression.
     *

     * NB: This is not valid until the CE has been parsed!
     */
    public final Enumeration getClauses() {
        return _cv.elements();
    }

    /**
     * This function sends the variables described in the constrained DDS to
     * the output described by sink. This function calls
     * parse_constraint(), BaseType::read(), and
     * ServerIO::serialize().
     *
     * @param dataset  The name of the dataset to send.
     * @param sink     A pointer to the output buffer for the data.
     * @param specialO An Object passed by the server. This is typically used by server implementations
     *                 to deliver needed functionaliy or information to the read methods of each BaseType.
     * @see #parseConstraint
     * @see ServerMethods#serialize(String, DataOutputStream,
     *      CEEvaluator, Object) ServerMethods.serialize()
     */
    public void send(String dataset, OutputStream sink, Object specialO)
            throws NoSuchVariableException, DAP2ServerSideException, IOException {

        Enumeration e = _dds.getVariables();
        while (e.hasMoreElements()) {

            ServerMethods s = (ServerMethods) e.nextElement();

            if (_Debug)
                System.out.println("Sending variable: " +
                        ((BaseType) s).getEncodedName());

            if (s.isProject()) {

                if (_Debug)
                    System.out.println("Calling "
                            + ((BaseType) s).getTypeName()
                            + ".serialize() (Name: "
                            + ((BaseType) s).getEncodedName()
                            + ")");

              //System.out.printf("serialize %s (%s) start=%d%n ", ((BaseType) s).getName(), ((BaseType) s).getTypeName(),
              //    ((DataOutputStream) sink).size());

                s.serialize(dataset, (DataOutputStream) sink, this, specialO);
            }
        }
      //System.out.printf("serialize total size=%d%n ", ((DataOutputStream) sink).size());

    }

    /**
     * Evaluate all of the Clauses in the Clause vector.
     *
     * @param specialO That special Object that can be passed down
     *                 through the DDS.send() method.
     * @return True if all the Clauses evaluate to true, false otherwise.
     */
    public boolean evalClauses(Object specialO) throws NoSuchVariableException, DAP2ServerSideException, IOException {

        boolean result = true;
        Enumeration ec = getClauses();

        while (ec.hasMoreElements() && result == true) {
            Object o = ec.nextElement();
            if (_Debug) {
                System.out.println("Evaluating clause: " + ec.nextElement());
            }

            result = ((TopLevelClause) o).evaluate();

        }

        // Hack: pop the projections of all DArrayDimensions that
        // have been pushed.

        return (result);
    }

    /**
     * Mark all the variables in the DDS either as part of the current
     * projection (when state is true) or not
     * (state is false). This is a convenience function that
     * provides a way to clear or set an entire dataset described by a DDS
     * with respect to its projection.
     *
     * @param state true if the variables should all be projected, false is
     *              no variable should be projected.
     */
    public void markAll(boolean state)
            throws DAP2Exception, NoSuchVariableException, SBHException
    {
        // For all the Variables in the DDS
        Enumeration e = _dds.getVariables();
        while (e.hasMoreElements()) {
            // Get the thing
            Object o = e.nextElement();
            //- Clip this to stop marking all dimensions of Grids and Arrays
            // If we are marking all for true, then we need to make sure
            // we get all the parts of each array and grid

            // This code should probably be moved into SDArray and SDGrid.
            // There we should add a resetProjections() method that changes
            // the current projection to be the entire array. 11/18/99 jhrg
            if (state) {
                if (o instanceof SDArray) { // Is this thing a SDArray?
                    SDArray SDA = (SDArray) o;

                    // Get it's DArrayDimensions
                    Enumeration eSDA = SDA.getDimensions();
                    while (eSDA.hasMoreElements()) {
                        DArrayDimension dad = (DArrayDimension) eSDA.nextElement();
                        // Tweak it's projection state
                        dad.setProjection(0, 1, dad.getSize() - 1);
                    }
                } else if (o instanceof SDGrid) {  // Is this thing a SDGrid?
                    SDGrid SDG = (SDGrid) o;
                    SDArray sdgA = (SDArray) SDG.getVar(0); // Get it's internal SDArray.

                    // Get it's DArrayDimensions
                    Enumeration eSDA = sdgA.getDimensions();
                    while (eSDA.hasMoreElements()) {
                        DArrayDimension dad = (DArrayDimension) eSDA.nextElement();
                        // Tweak it's projection state
                        dad.setProjection(0, 1, dad.getSize() - 1);

                    }
                }
            }
            //-------------------------- End Clip ---------------------------

            ServerMethods s = (ServerMethods) o;
            s.setProject(state);
        }
    }

    /**
         * Print all of the Clauses in the Clause vector.
         *
         */
        public void printConstraint(PrintWriter pw)
        {
            Enumeration ec = getClauses();
            boolean first = true;
            while (ec.hasMoreElements()) {
                Clause cl = (Clause)ec.nextElement();
                if(!first) pw.print(" & ");
                cl.printConstraint(pw);
                first = false;
            }
            pw.flush();
        }
}