/*
 * Copyright (c) 1998 - 2009. University Corporation for Atmospheric Research/Unidata
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package ucar.nc2.ui.point;

import ucar.nc2.ui.util.Renderer;
import ucar.unidata.geoloc.*;
import ucar.nc2.ft.PointFeature;

import java.util.ArrayList;
import java.awt.*;
import java.awt.geom.Rectangle2D;
import java.awt.geom.AffineTransform;
import java.awt.geom.GeneralPath;
import java.awt.geom.Point2D;
import java.io.IOException;

import ucar.nc2.ui.widget.FontUtil;

/**
 * Describe
 *
 * @author caron
 * @since Nov 17, 2009
 */
public class PointRenderer implements Renderer {

  private java.util.List<ObservationUI> obsUIlist = new ArrayList<ObservationUI>(); // ObservationUI objects
  private ProjectionImpl project = null;        // display projection

  // drawing parameters
  private Color color = Color.black;
  private Color selectedColor = Color.magenta;
  private int circleRadius = 3;
  private Rectangle2D circleBB = new Rectangle2D.Double(-circleRadius, -circleRadius, 2*circleRadius, 2*circleRadius);
  private FontUtil.StandardFont textFont;
  private boolean drawConnectingLine = false;

  // working objects to minimize excessive gc
 // private Point2D.Double ptN = new Point2D.Double();

  private ObservationUI selected = null;

  // misc state
  private boolean declutter = true;
  private boolean posWasCalc = false;
  //private boolean debug = false;

    /** constructor */
  public PointRenderer () {
    textFont = FontUtil.getStandardFont(10);
  }

  public void incrFontSize() { textFont.incrFontSize(); };
  public void decrFontSize() { textFont.decrFontSize(); };

  public void setColor(java.awt.Color color){ this.color = color; }
  public java.awt.Color getColor() { return color; }

  public LatLonRect getPreferredArea() {
    return null;
  }

  public void setPointFeatures( java.util.List<PointFeature> obs) throws IOException {
    obsUIlist = new ArrayList<ObservationUI>( obs.size());
    for (PointFeature ob : obs) {
      ObservationUI sui = new ObservationUI( ob); // wrap in a StationUI
      obsUIlist.add(sui); // wrap in a StationUI
    }
    posWasCalc = false;
    calcWorldPos();
    selected = null;
  }

  public void setDrawConnectingLine(boolean drawConnectingLine) {
    this.drawConnectingLine = drawConnectingLine;
  }

  public void setSelected( PointFeature obs) {
    selected = null;
    for (int i=0; i < obsUIlist.size(); i++) {
      ObservationUI s = (ObservationUI) obsUIlist.get(i);
      if (testPointObsDatatype( s.obs,  obs)) {
        selected = s;
        break;
      }
    }
  }

  private boolean testPointObsDatatype(PointFeature obs1, PointFeature obs2) {
    if (obs1.getObservationTime() != obs2.getObservationTime())
      return false;
    ucar.unidata.geoloc.EarthLocation loc1 = obs1.getLocation();
    ucar.unidata.geoloc.EarthLocation loc2 = obs2.getLocation();
    if (loc1.getLatitude() != loc2.getLatitude())
      return false;
    if (loc1.getLongitude() != loc2.getLongitude())
      return false;
    return true;
  }

  public void setDeclutter(boolean declut) { declutter = declut; }
  public boolean getDeclutter() { return declutter; }

  public void setProjection(ProjectionImpl project) {
    this.project = project;
    calcWorldPos();
  }

  private void calcWorldPos() {
    if (project == null) return;
    for (int i = 0; i < obsUIlist.size(); i++) {
      ObservationUI s = (ObservationUI) obsUIlist.get(i);
      s.worldPos.setLocation( project.latLonToProj(s.latlonPos));
    }
    posWasCalc = true;
  }

    ///////////////// pickable stuff

  /*
   * Find station that contains this point. If it exists, make it the
   *   "selected" station.
   * @param pickPt: world coordinates
   * @return station that contains this point, or null if none.
   *
  public PointObsDatatype pick(Point2D pickPt) {
    if (world2Normal == null || pickPt == null || obsUIlist.isEmpty()) return null;

    world2Normal.transform(pickPt, ptN); // work in normalized coordinate space
    ObservationUI closest = (ObservationUI) stationGrid.findIntersection(ptN);
    setSelectedStation( closest);

    return getSelectedStation();
  }

  /*
   * Find station closest to this point. Make it the "selected" station.
   * @param pickPt: world coordinates
   * @return station that contains this point, or null if none.
   *
  public PointObsDatatype pickClosest(Point2D pickPt) {
    if (world2Normal == null || pickPt == null || obsUIlist.isEmpty()) return null;

    world2Normal.transform(pickPt, ptN); // work in normalized coordinate space
    ObservationUI closest = (ObservationUI) stationGrid.findClosest(ptN);

    if (debug) System.out.println("closest= " +closest);
    setSelectedStation( closest);

    return getSelectedStation();
  }

  /**
   * Get the selected station, or null.
   *
  public PointObsDatatype getSelectedStation() {
    return (selected != null) ? selected.getObservation() : null;
  }

  /**
   * Is this point contained in a Station bounding box?
   * @param p: normalized coords.
   * @return Station containing pt, or null.
   *
  public ObservationUI isOnStation( Point p) {
    return (ObservationUI) stationGrid.findIntersection (p);
  } */

  public void draw(java.awt.Graphics2D g, java.awt.geom.AffineTransform normal2Device) {
    if ((project == null) || !posWasCalc) return;

    // use world coordinates for position, but draw in screen coordinates
    // so that the symbols stay the same size
    AffineTransform world2Device = g.getTransform();
    g.setTransform(normal2Device);        //  identity transform for screen coords

    // transform World to Normal coords:
    //    world2Normal = pixelAT-1 * world2Device
    // cache for pick closest
    AffineTransform world2Normal;
    try {
      world2Normal = normal2Device.createInverse();
      world2Normal.concatenate( world2Device);
    } catch ( java.awt.geom.NoninvertibleTransformException e) {
      System.out.println(
            " RendSurfObs: NoninvertibleTransformException on " +
            normal2Device);
      return;
    }

    // we want aliasing; but save previous state to restore at end
    Object saveHint = g.getRenderingHint(RenderingHints.KEY_ANTIALIASING);
    //g.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
    //    RenderingHints.VALUE_ANTIALIAS_ON);
    g.setStroke(new java.awt.BasicStroke(1.0f));

    /* set up the grid
    Rectangle2D bbox = (Rectangle2D) g.getClip(); // clipping area in normal coords
    // clear the grid = "no stations are drawn"
    stationGrid.clear();
    // set the grid size based on typical bounding box
    stationGrid.setGrid(bbox, typicalBB.getWidth(), typicalBB.getHeight());

    // always draw selected
    if (selected != null) {
      selected.calcPos( world2Normal);
      stationGrid.markIfClear( selected.getBB(), selected);
      selected.draw(g);
    } */

    g.setFont( textFont.getFont());
    g.setColor(color);

    int count = 0;
    int npts = obsUIlist.size();
    GeneralPath path = new GeneralPath(GeneralPath.WIND_EVEN_ODD, npts);
    for (int i=0; i < npts; i++) {
      ObservationUI s = (ObservationUI) obsUIlist.get(i);
      s.calcPos(world2Normal);
      s.draw(g);

      if (Double.isNaN( s.screenPos.getX())) {
        System.out.println("screenPos="+s.screenPos+" world = "+s.worldPos);
        continue;
      }

      if (count == 0)
        path.moveTo( (float) s.screenPos.getX(), (float) s.screenPos.getY());
      else
        path.lineTo( (float) s.screenPos.getX(), (float) s.screenPos.getY());
      count++;
    }

    g.setColor(color);
    if (drawConnectingLine)
      g.draw( path);

    // draw selected
    if (selected != null)
      selected.draw(g);

    // restore
    g.setTransform(world2Device);
    g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, saveHint);
  }

  public class ObservationUI {
    private PointFeature obs;

    private LatLonPointImpl latlonPos = new LatLonPointImpl();       // latlon pos
    private ProjectionPointImpl worldPos = new ProjectionPointImpl();// world pos
    private Point2D.Double screenPos = new Point2D.Double(); // normalized screen pos

    private Rectangle2D bb;       // bounding box, in normalized screen coords, loc = 0, 0
    private Rectangle2D bbPos = new Rectangle2D.Double();    // bounding box, translated to current drawing pos
    // private boolean selected = false;

    ObservationUI(PointFeature obs) {
      this.obs = obs;
      latlonPos.setLatitude(obs.getLocation().getLatitude());
      latlonPos.setLongitude(obs.getLocation().getLongitude());

      // text bb
      Dimension t = textFont.getBoundingBox("O"); // LOOK temp
      bb = new Rectangle2D.Double(-circleRadius, -circleRadius-t.getHeight(), t.getWidth(), t.getHeight());
      // add circle bb
      bb.add( circleBB);
    }

    public PointFeature getObservation() { return obs; }
    public LatLonPoint getLatLon() { return latlonPos; }
    public ProjectionPointImpl getLocation() { return worldPos; }
    //public boolean isSelected() { return selected; }
    //public void setIsSelected(boolean selected) { this.selected = selected; }
    public Rectangle2D getBB() { return bbPos; }

    boolean contains( Point p) {
      return bbPos.contains( p);
    }

    void calcPos( AffineTransform w2n) {
      w2n.transform( new Point2D.Double(worldPos.getX(), worldPos.getY()), screenPos);  // work in normalized coordinate space
      bbPos.setRect( screenPos.getX() + bb.getX(), screenPos.getY() + bb.getY(),
          bb.getWidth(), bb.getHeight());
    }

    void draw(Graphics2D g) {
      if (this == selected) {
        g.setColor( selectedColor);
        fillCircle(g, screenPos);
        g.setColor( color);
      } else
        drawCircle(g, screenPos);
      //drawText(g, screenPos, id);
    }

    /** draw the symbol at the specified location
     * @param g Graphics2D object
     * @param loc the reference point (center of the CloudSymbol icon offset from here)
     */
    private void drawCircle( Graphics2D g, Point2D loc) {
      int x = (int) (loc.getX()-circleRadius);
      int y = (int) (loc.getY()-circleRadius);
      g.drawOval( x, y, 2*circleRadius, 2*circleRadius);
    }

    private void fillCircle( Graphics2D g, Point2D loc) {
      int x = (int) (loc.getX()-circleRadius);
      int y = (int) (loc.getY()-circleRadius);
      g.fillOval( x, y, 2*circleRadius, 2*circleRadius);
    }

    /* private void drawText( Graphics2D g, Point2D loc, String text) {
      int x = (int) (loc.getX() - circleRadius);
      int y = (int) (loc.getY() - circleRadius);
      g.drawString( text, x, y);
    } */

  } // end inner class ObservationUI

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.thredds;

import ucar.nc2.*;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.stream.CdmRemote;
import ucar.nc2.ft.remote.CdmrFeatureDataset;
import ucar.nc2.ft.FeatureDatasetFactoryManager;
import ucar.nc2.constants.FeatureType;

import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.ncml.NcMLReader;

import thredds.catalog.*;
import ucar.nc2.util.Optional;
import ucar.unidata.util.StringUtil2;

import java.util.List;
import java.util.ArrayList;
import java.util.Formatter;
import java.io.IOException;

/**
 * This tries to translate a THREDDS InvDataset into a data object that can be used, either as a NetcdfDataset or as a FeatureDataset.
 *

 * As input, it can take
 *

 An InvAccess object.
 *
 An InvDataset object. If the InvDataset has more that one InvAccess, it has to try to choose which to use,
 * based on what Service type we know how to work with.
 *
 A url of the form [thredds:]catalog.xml#datasetId. In this case it opens the catalog, and looks for the
 * InvDataset with the given datasetId.
 *
 A url of the form thredds:resolve:resolveURL. In this case it expects that the URL will return a catalog with a
 * single top level dataset, which is the "resolved" dataset.
 *

 *

 * It annotates the NetcdfDataset with info from the InvDataset.
 *

 * You can reuse a ThreddsDataFactory, but only within a single thread.
 *
 * @author caron
 */
public class ThreddsDataFactory {
  static public final String PROTOCOL = "thredds";
  static public final String SCHEME = PROTOCOL + ":";
  static private ServiceType[] preferAccess;

  static public void setPreferCdm(boolean prefer) {
    preferAccess = prefer ? new ServiceType[] {ServiceType.CdmRemote} : null;
  }
  static public void setPreferAccess(ServiceType... prefer) {
    preferAccess = prefer;
  }

  static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debugOpen = debugFlag.isSet("thredds/debugOpen");
    debugTypeOpen = debugFlag.isSet("thredds/openDatatype");
  }

  static private boolean debugOpen = false;
  static private boolean debugTypeOpen = false;

  /**
   * The result of trying to open a THREDDS dataset.
   * If fatalError is true, the operation failed, errLog should indicate why.
   * Otherwise, the FeatureType and FeatureDataset is valid.
   * There may still be warning or diagnostic errors in errLog.
   */
  public static class Result {
    public boolean fatalError;
    public Formatter errLog = new Formatter();

    public FeatureType featureType;
    public FeatureDataset featureDataset;
    public String imageURL;

    public String location;
    public InvAccess accessUsed;

    @Override
    public String toString() {
      final StringBuilder sb = new StringBuilder();
      sb.append("Result");
      sb.append("{fatalError=").append(fatalError);
      sb.append(", errLog=").append(errLog);
      sb.append(", featureType=").append(featureType);
      sb.append(", featureDataset=").append(featureDataset);
      sb.append(", imageURL='").append(imageURL).append('\'');
      sb.append(", location='").append(location).append('\'');
      sb.append(", accessUsed=").append(accessUsed);
      sb.append('}');
      return sb.toString();
    }
  }

  /**
   * Open a FeatureDataset from a URL location string. Example URLS:

   *
http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:file:c:/test/data/catalog/addeStationDataset.xml#AddeSurfaceData (absolute file)
   *
thredds:resolve:resolveURL
   *

   *
   * @param urlString [thredds:]catalog.xml#datasetId
   * @param task      may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws java.io.IOException on read error
   */
  public ThreddsDataFactory.Result openFeatureDataset(String urlString, ucar.nc2.util.CancelTask task) throws IOException {

    ThreddsDataFactory.Result result = new ThreddsDataFactory.Result();
    InvDataset invDataset = processLocation(urlString, task, result);
    if (result.fatalError)
      return result;

    return openFeatureDataset(null, invDataset, task, result);
  }

  /**
   * Open a FeatureDataset from a URL location string, and a desired type (may by NONE or null).
   *
   * @param wantFeatureType desired feature type, may be NONE or null
   * @param urlString       [thredds:]catalog.xml#datasetId
   * @param task            may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws java.io.IOException on read error
   */
  public ThreddsDataFactory.Result openFeatureDataset(FeatureType wantFeatureType, String urlString, ucar.nc2.util.CancelTask task)
          throws IOException {
    ThreddsDataFactory.Result result = new ThreddsDataFactory.Result();
    InvDataset invDataset = processLocation(urlString, task, result);
    if (result.fatalError || invDataset == null)
      return result;

    return openFeatureDataset(wantFeatureType, invDataset, task, result);
  }

  private InvDataset processLocation(String location, ucar.nc2.util.CancelTask task, Result result) {
    location = location.trim();
    location = StringUtil2.replace(location, '\\', "/");

    if (location.startsWith(SCHEME))
      location = location.substring(8);

    if (location.startsWith("resolve:")) {
      location = location.substring(8);
      return openResolver(location, task, result);
    }

    if (!location.startsWith("http:") && !location.startsWith("file:"))   // LOOK whats this for??
      location = "http:" + location;

    InvCatalog catalog;
    InvDataset invDataset;
    String datasetId;

    int pos = location.indexOf('#');
    if (pos < 0) {
      result.fatalError = true;
      result.errLog.format("Must have the form catalog.xml#datasetId%n");
      return null;
    }

    InvCatalogFactory catFactory = new InvCatalogFactory("", false);
    String catalogLocation = location.substring(0, pos);
    catalog = catFactory.readXML(catalogLocation);
    StringBuilder buff = new StringBuilder();
    if (!catalog.check(buff)) {
      result.errLog.format("Invalid catalog from Resolver <%s>%n%s%n", catalogLocation, buff.toString());
      result.fatalError = true;
      return null;
    }

    datasetId = location.substring(pos + 1);
    invDataset = catalog.findDatasetByID(datasetId);
    if (invDataset == null) {
      result.fatalError = true;
      result.errLog.format("Could not find dataset %s in %s %n", datasetId, catalogLocation);
      return null;
    }

    return invDataset;
  }

  /**
   * Open a FeatureDataset from an InvDataset object, deciding on which InvAccess to use.
   *
   * @param invDataset use this to figure out what type, how to open, etc
   * @param task       allow user to cancel; may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws IOException on read error
   */
  public ThreddsDataFactory.Result openFeatureDataset(InvDataset invDataset, ucar.nc2.util.CancelTask task) throws IOException {
    return openFeatureDataset(null, invDataset, task, new Result());
  }

  public ThreddsDataFactory.Result openFeatureDataset(FeatureType wantFeatureType, InvDataset invDataset, ucar.nc2.util.CancelTask task, Result result)
          throws IOException {

    result.featureType = invDataset.getDataType();
    if (result.featureType == null)
      result.featureType = wantFeatureType;

    // look for remote FeatureDataset
    if ((result.featureType != null) && result.featureType.isPointFeatureType()) {
      InvAccess access = findAccessByServiceType(invDataset.getAccess(), ServiceType.CdmrFeature);
      if (access != null)
        return openFeatureDataset(result.featureType, access, task, result);
    }

    // special handling for images
    if (result.featureType == FeatureType.IMAGE) {
      InvAccess access = getImageAccess(invDataset, task, result);
      if (access != null) {
        return openFeatureDataset(result.featureType, access, task, result);
      } else
        result.fatalError = true;
      return result;
    }

    // special handling for DQC
    InvAccess qc = invDataset.getAccess(ServiceType.QC);
    if (qc != null) {

      if (result.featureType == FeatureType.STATION) {

        /* DqcFactory dqcFactory = new DqcFactory(true);
        String dqc_location = qc.getStandardUrlName();
        QueryCapability dqc = dqcFactory.readXML(dqc_location);
        if (dqc.hasFatalError()) {
          result.errLog.append(dqc.getErrorMessages());
          result.fatalError = true;
        } */

        result.featureDataset = null; // LOOK FIX ucar.nc2.thredds.DqcStationObsDataset.factory(invDataset, dqc_location, result.errLog);
        result.fatalError = (result.featureDataset == null);

      } else {
        result.errLog.format("DQC must be station DQC, dataset = %s %n", invDataset.getName());
        result.fatalError = true;
      }

      return result;
    }

    NetcdfDataset ncd = openDataset(invDataset, true, task, result);
    if (null != ncd)
      result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);

    if (null == result.featureDataset)
      result.fatalError = true;
    else {
      result.location = result.featureDataset.getLocation();
      if ((result.featureType == null) && (result.featureDataset != null))
        result.featureType = result.featureDataset.getFeatureType();
    }

    return result;
  }

  /**
   * Open a FeatureDataset from an InvAccess object.
   *
   * @param access use this InvAccess.
   * @param task   may be null
   * @return ThreddsDataFactory.Result check fatalError for validity
   * @throws IOException on read error
   */
  public ThreddsDataFactory.Result openFeatureDataset(InvAccess access, ucar.nc2.util.CancelTask task) throws IOException {
    InvDataset invDataset = access.getDataset();
    ThreddsDataFactory.Result result = new Result();
    if (invDataset.getDataType() == null) {
      result.errLog.format("InvDatasert must specify a FeatureType%n");
      result.fatalError = true;
      return result;
    }

    return openFeatureDataset(invDataset.getDataType(), access, task, result);
  }

  private ThreddsDataFactory.Result openFeatureDataset(FeatureType wantFeatureType, InvAccess access, ucar.nc2.util.CancelTask task, Result result)
          throws IOException {
    result.featureType = wantFeatureType;
    result.accessUsed = access;

    // special handling for IMAGE
    if (result.featureType == FeatureType.IMAGE) {
      result.imageURL = access.getStandardUrlName();
      result.location = result.imageURL;
      return result;
    }

    if (access.getService().getServiceType() == ServiceType.CdmrFeature) {
      Optional<FeatureDataset> opt = CdmrFeatureDataset.factory(wantFeatureType, access.getStandardUrlName());
      if (opt.isPresent())
        result.featureDataset = opt.get();
      else
        result.errLog.format("%s", opt.getErrorMessage());

    } else {

      // all other datatypes
      NetcdfDataset ncd = openDataset(access, true, task, result);
      if (null != ncd) {
        result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);
      }
    }

    if (null == result.featureDataset)
      result.fatalError = true;
    else {
      result.location = result.featureDataset.getLocation();
      result.featureType = result.featureDataset.getFeatureType();
    }

    return result;
  }

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Open a NetcdfDataset from a URL location string. Example URLS:

   *
http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:http://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds://localhost:8080/test/addeStationDataset.xml#surfaceHourly
   *
thredds:file:c:/dev/netcdf-java-2.2/test/data/catalog/addeStationDataset.xml#AddeSurfaceData (absolute file)
   *
thredds:resolve:resolveURL
   *

   *
   * @param location catalog.xml#datasetId, may optionally start with "thredds:"
   * @param task     may be null
   * @param log      error messages gp here, may be null
   * @param acquire  if true, aquire the dataset, else open it
   * @return NetcdfDataset
   * @throws java.io.IOException on read error
   */
  public NetcdfDataset openDataset(String location, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log) throws IOException {
    Result result = new Result();
    InvDataset invDataset = processLocation(location, task, result);
    if (result.fatalError || invDataset == null) {
      if (log != null) log.format("%s", result.errLog);
      return null;
    }

    return openDataset(invDataset, acquire, task, result);
  }

  /**
   * Try to open as a NetcdfDataset.
   *
   * @param invDataset open this
   * @param acquire    if true, aquire the dataset, else open it
   * @param task       may be null
   * @param log        error message, may be null
   * @return NetcdfDataset or null if failure
   * @throws IOException on read error
   */
  public NetcdfDataset openDataset(InvDataset invDataset, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log) throws IOException {
    Result result = new Result();
    NetcdfDataset ncd = openDataset(invDataset, acquire, task, result);
    if (log != null) log.format("%s", result.errLog);
    return (result.fatalError) ? null : ncd;
  }

  private NetcdfDataset openDataset(InvDataset invDataset, boolean acquire, ucar.nc2.util.CancelTask task, Result result) throws IOException {

    IOException saveException = null;

    List<InvAccess> accessList = new ArrayList<>(invDataset.getAccess()); // a list of all the accesses
    while (accessList.size() > 0) {
      InvAccess access = chooseDatasetAccess(accessList);

      // no valid access
      if (access == null) {
        result.errLog.format("No access that could be used in dataset %s %n", invDataset);
        if (saveException != null)
          throw saveException;
        return null;
      }

      String datasetLocation = access.getStandardUrlName();
      ServiceType serviceType = access.getService().getServiceType();
      if (debugOpen)
        System.out.println("ThreddsDataset.openDataset try " + datasetLocation + " " + serviceType);

      // deal with RESOLVER type
      if (serviceType == ServiceType.RESOLVER) {
        InvDatasetImpl rds = openResolver(datasetLocation, task, result);
        if (rds == null) return null;
        accessList = new ArrayList<>(rds.getAccess());
        continue;
      }

      // ready to open it through netcdf API
      NetcdfDataset ds;

// try to open
      try {
        ds = openDataset(access, acquire, task, result);

      } catch (IOException e) {
        result.errLog.format("Cant open %s %n err=%s%n", datasetLocation, e.getMessage());
        if (debugOpen) {
          System.out.println("Cant open= " + datasetLocation + " " + serviceType);
          e.printStackTrace();
        }

        accessList.remove(access);
        saveException = e;
        continue;
      }

      result.accessUsed = access;
      return ds;
    } // loop over accesses

    if (saveException != null) throw saveException;
    return null;
  }

  /**
   * Try to open invAccess as a NetcdfDataset.
   *
   * @param access  open this InvAccess
   * @param acquire if true, aquire the dataset, else open it
   * @param task    may be null
   * @param log     error message, may be null
   * @return NetcdfDataset or null if failure
   * @throws IOException on read error
   */
  public NetcdfDataset openDataset(InvAccess access, boolean acquire, ucar.nc2.util.CancelTask task, Formatter log) throws IOException {
    Result result = new Result();
    NetcdfDataset ncd = openDataset(access, acquire, task, result);
    if (log != null) log.format("%s", result.errLog);
    return (result.fatalError) ? null : ncd;
  }

  private static boolean enhanceMode = false;

  private NetcdfDataset openDataset(InvAccess access, boolean acquire, ucar.nc2.util.CancelTask task, Result result) throws IOException {
    InvDataset invDataset = access.getDataset();
    String datasetId = invDataset.getID();
    String title = invDataset.getName();

    String datasetLocation = access.getStandardUrlName();
    ServiceType serviceType = access.getService().getServiceType();
    if (debugOpen) System.out.println("ThreddsDataset.openDataset= " + datasetLocation);

    // deal with RESOLVER type
    if (serviceType == ServiceType.RESOLVER) {
      InvDatasetImpl rds = openResolver(datasetLocation, task, result);
      if (rds == null) return null;
      return openDataset(rds, acquire, task, result);
    }

    // ready to open it through netcdf API
    NetcdfDataset ds;

    // open DODS type
    String prefix = null;
    if ((serviceType == ServiceType.OPENDAP)|| (serviceType == ServiceType.DODS))
        prefix = "dods:";
    else if(serviceType == ServiceType.DAP4)
        prefix = "dap4:";
    if(prefix != null) {
      String curl = datasetLocation;
      if (curl.startsWith("http:")) {
          curl = prefix + datasetLocation.substring(5);
      } else if (curl.startsWith("https:")) {
          curl =  prefix + curl.substring(6);
      }
      ds = acquire ? NetcdfDataset.acquireDataset(DatasetUrl.findDatasetUrl(curl), enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    // open CdmRemote
    else if (serviceType == ServiceType.CdmRemote) {
      String curl = CdmRemote.canonicalURL(datasetLocation);
      ds = acquire ? NetcdfDataset.acquireDataset(DatasetUrl.findDatasetUrl(curl), enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    // open CdmRemote
    else if ((serviceType == ServiceType.HTTP) || (serviceType == ServiceType.HTTPServer)) {
      String curl =  (datasetLocation.startsWith("http:")) ? "httpserver::" + datasetLocation.substring(5) : datasetLocation;
      ds = acquire ? NetcdfDataset.acquireDataset(DatasetUrl.findDatasetUrl(curl), enhanceMode, task) : NetcdfDataset.openDataset(curl, enhanceMode, task);
    }

    else {
      // open through NetcdfDataset API
      ds = acquire ? NetcdfDataset.acquireDataset(DatasetUrl.findDatasetUrl(datasetLocation), enhanceMode, task) : NetcdfDataset.openDataset(datasetLocation, enhanceMode, task);
    }

    result.accessUsed = access;
    if (ds == null) return null;

    ds.setId(datasetId);
    ds.setTitle(title);
    annotate(invDataset, ds);

    // see if there's metadata LOOK whats this
    List<InvMetadata> list = invDataset.getMetadata(MetadataType.NcML);
    if (list.size() > 0) {
      InvMetadata ncmlMetadata = list.get(0);
      NcMLReader.wrapNcML(ds, ncmlMetadata.getXlinkHref(), null);
    }

    return ds;
  }

  /**
   * Find the "best" access in case theres more than one, based on what the CDM knows
   * how to open and use.
   *
   * @param accessList choose from this list.
   * @return best access method.
   */
  public InvAccess chooseDatasetAccess(List<InvAccess> accessList) {
    if (accessList.size() == 0)
      return null;

    InvAccess access = null;
    if (preferAccess != null) {
      for (ServiceType type : preferAccess) {
        access = findAccessByServiceType(accessList, type);
        if (access != null) break;
      }
    }

    // the order indicates preference
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.CdmRemote);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.DODS);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.OPENDAP);
    if (access == null)
       access = findAccessByServiceType(accessList, ServiceType.DAP4);
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.FILE); // should mean that it can be opened through netcdf API
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.NETCDF); //  ServiceType.NETCDF is deprecated, use FILE

    // look for HTTP with format we can read
    if (access == null) {
      InvAccess tryAccess = findAccessByServiceType(accessList, ServiceType.HTTPServer);
      if (tryAccess == null)
        tryAccess = findAccessByServiceType(accessList, ServiceType.HTTP); //  ServiceType.HTTP should be HTTPServer

      if (tryAccess != null) {
        DataFormatType format = tryAccess.getDataFormatType();

        // these are the file types we can read
        if ((DataFormatType.BUFR == format) || (DataFormatType.GINI == format) || (DataFormatType.GRIB1 == format)
                || (DataFormatType.GRIB2 == format) || (DataFormatType.HDF5 == format) || (DataFormatType.NCML == format)
                || (DataFormatType.NETCDF == format) || (DataFormatType.NEXRAD2 == format) || (DataFormatType.NIDS == format)) {
          access = tryAccess;
        }
      }
    }

    // ADDE
    if (access == null)
      access = findAccessByServiceType(accessList, ServiceType.ADDE);

    // RESOLVER
    if (access == null) {
      access = findAccessByServiceType(accessList, ServiceType.RESOLVER);
    }

    return access;
  }

  private InvDatasetImpl openResolver(String urlString, ucar.nc2.util.CancelTask task, Result result) {
    InvCatalogFactory catFactory = new InvCatalogFactory("", false);
    InvCatalogImpl catalog = catFactory.readXML(urlString);
    if (catalog == null) {
      result.errLog.format("Couldnt open Resolver %s %n ", urlString);
      return null;
    }
    StringBuilder buff = new StringBuilder();
    if (!catalog.check(buff)) {
      result.errLog.format("Invalid catalog from Resolver <%s>%n%s%n", urlString, buff.toString());
      result.fatalError = true;
      return null;
    }
    InvDataset top = catalog.getDataset();
    if (top.hasAccess())
      return (InvDatasetImpl) top;
    else {
      java.util.List datasets = top.getDatasets();
      return (InvDatasetImpl) datasets.get(0);
    }
  }

  /**
   * Add information from the InvDataset to the NetcdfDataset.
   *
   * @param ds        get info from here
   * @param ncDataset add to here
   */
  public static void annotate(InvDataset ds, NetcdfDataset ncDataset) {
    ncDataset.setTitle(ds.getName());
    ncDataset.setId(ds.getID());

// add properties as global attributes
    for (InvProperty p : ds.getProperties()) {
      String name = p.getName();
      if (null == ncDataset.findGlobalAttribute(name)) {
        ncDataset.addAttribute(null, new Attribute(name, p.getValue()));
      }
    }

    /* ThreddsMetadata.GeospatialCoverage geoCoverage = ds.getGeospatialCoverage();
   if (geoCoverage != null) {
     if ( null != geoCoverage.getNorthSouthRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_lat_min", new Double(geoCoverage.getLatSouth())));
       ncDataset.addAttribute(null, new Attribute("geospatial_lat_max", new Double(geoCoverage.getLatNorth())));
     }
     if ( null != geoCoverage.getEastWestRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_lon_min", new Double(geoCoverage.getLonWest())));
       ncDataset.addAttribute(null, new Attribute("geospatial_lon_max", new Double(geoCoverage.getLonEast())));
     }
     if ( null != geoCoverage.getUpDownRange()) {
       ncDataset.addAttribute(null, new Attribute("geospatial_vertical_min", new Double(geoCoverage.getHeightStart())));
       ncDataset.addAttribute(null, new Attribute("geospatial_vertical_max", new Double(geoCoverage.getHeightStart() + geoCoverage.getHeightExtent())));
     }
   }

   DateRange timeCoverage = ds.getTimeCoverage();
   if (timeCoverage != null) {
     ncDataset.addAttribute(null, new Attribute("time_coverage_start", timeCoverage.getStart().toDateTimeStringISO()));
     ncDataset.addAttribute(null, new Attribute("time_coverage_end", timeCoverage.getEnd().toDateTimeStringISO()));
   } */

    ncDataset.finish();
  }

//////////////////////////////////////////////////////////////////////////////////
// image

// look for an access method for an image datatype

  private InvAccess getImageAccess(InvDataset invDataset, ucar.nc2.util.CancelTask task, Result result) {

    List<InvAccess> accessList = new ArrayList<>(invDataset.getAccess()); // a list of all the accesses
    while (accessList.size() > 0) {
      InvAccess access = chooseImageAccess(accessList);
      if (access != null) return access;

// next choice is resolver type.
      access = invDataset.getAccess(ServiceType.RESOLVER);

// no valid access
      if (access == null) {
        result.errLog.format("No access that could be used for Image Type %s %n", invDataset);
        return null;
      }

      // deal with RESOLVER type
      String datasetLocation = access.getStandardUrlName();
      InvDatasetImpl rds = openResolver(datasetLocation, task, result);
      if (rds == null)
        return null;

// use the access list from the resolved dataset
      accessList = new ArrayList<>(invDataset.getAccess());
    } // loop over accesses

    return null;
  }

  private InvAccess chooseImageAccess(List<InvAccess> accessList) {
    InvAccess access;

    access = findAccessByDataFormatType(accessList, DataFormatType.JPEG);
    if (access != null) return access;

    access = findAccessByDataFormatType(accessList, DataFormatType.GIF);
    if (access != null) return access;

    access = findAccessByDataFormatType(accessList, DataFormatType.TIFF);
    if (access != null) return access;

    access = findAccessByServiceType(accessList, ServiceType.ADDE);
    if (access != null) {
      String datasetLocation = access.getStandardUrlName();
      if (datasetLocation.indexOf("image") > 0)
        return access;
    }

    return access;
  }

///////////////////////////////////////////////////////////////////

// works against the accessList instead of the dataset list, so we can remove and try again

  private InvAccess findAccessByServiceType(List<InvAccess> accessList, ServiceType type) {
    for (InvAccess a : accessList) {
      if (type.toString().equalsIgnoreCase(a.getService().getServiceType().toString()))
        return a;
    }
    return null;
  }

// works against the accessList instead of the dataset list, so we can remove and try again

  private InvAccess findAccessByDataFormatType(List<InvAccess> accessList, DataFormatType type) {
    for (InvAccess a : accessList) {
      if (type.toString().equalsIgnoreCase(a.getDataFormatType().toString()))
        return a;
    }
    return null;
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.client.catalog;

import ucar.nc2.Attribute;
import ucar.nc2.AttributeContainer;

import javax.annotation.concurrent.Immutable;
import java.util.ArrayList;
import java.util.List;

/**
 * Client catalog name/value pair
 *
 * @author caron
 * @since 1/7/2015
 */
@Immutable
public class Property {
  private final String name;
  private final String value;

  public Property(String name, String value) {
    this.name = name;
    this.value = value;
  }

  public String getName() {
    return name;
  }

  public String getValue() {
    return value;
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;

    Property property = (Property) o;

    if (!name.equals(property.name)) return false;
    return !(value != null ? !value.equals(property.value) : property.value != null);
  }

  @Override
  public int hashCode() {
    int result = name.hashCode();
    result = 31 * result + (value != null ? value.hashCode() : 0);
    return result;
  }

  // first one override
  public static List<Property> removeDups(List<Property> org) {
    List<Property> result = new ArrayList<>(org.size());
    for (Property p : org)
      if (!result.contains(p))  // O(n**2)
        result.add(p);
    return result;
  }

  public static List<Property> convertToProperties(AttributeContainer from) {
    List<Property> result = new ArrayList<>();
    for (Attribute p : from.getAttributes())
      result.add(new Property(p.getShortName(), p.getStringValue()));
    return result;
  }

  @Override
  public String toString() {
    return "Property{" +
            "name='" + name + '\'' +
            ", value='" + value + '\'' +
            '}';
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.collection;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.inventory.DateExtractor;
import thredds.inventory.MCollection;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.grib.GribIndexCache;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.util.cache.SmartArrayInt;
import ucar.unidata.util.StringUtil2;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.*;

/**
 * Collection of GribCollections or other PartitionCollections, partitioned by reference time.
 *
 * @author John
 * @since 12/7/13
 */
public class PartitionCollectionMutable extends GribCollectionMutable {

  static class PartitionForVariable2D {
    final int partno;
    final int groupno;
    final int varno; // , flag;     // what the hell is the flag used for ?
    GribCollectionMutable.VariableIndex vi;

    PartitionForVariable2D(int partno, int groupno, int varno) {
      this.partno = partno;
      this.groupno = groupno;
      this.varno = varno;
    }
  }

  public class VariableIndexPartitioned extends GribCollectionMutable.VariableIndex {
    final int nparts;
    SmartArrayInt partnoSA;
    SmartArrayInt groupnoSA;
    SmartArrayInt varnoSA;

    List<PartitionForVariable2D> partList; // used only when creating, then discarded in finish

    VariableIndexPartitioned(GroupGC g, VariableIndex other, int nparts) {
      super(g, other);
      this.nparts = nparts;
    }

    public void setPartitions(List<GribCollectionProto.PartitionVariable> pvList) {
      int[] partno = new int[nparts];
      int[] groupno = new int[nparts];
      int[] varno = new int[nparts];
      int count = 0;
      for (GribCollectionProto.PartitionVariable part : pvList) {
        partno[count] = part.getPartno();
        groupno[count] = part.getGroupno();
        varno[count] = part.getVarno();
        count++;
      }
      this.partnoSA =  new SmartArrayInt(partno);
      this.groupnoSA =  new SmartArrayInt(groupno);
      this.varnoSA =  new SmartArrayInt(varno);

      partList = null; // GC
    }

    public void finish() {
      if (partList == null) return;  // nothing to do
      if (partList.size() > nparts)  // might be smaller due to failed partition
        logger.warn("PartitionCollectionMutable partList.size() > nparts");

      int[] partno = new int[nparts];
      int[] groupno = new int[nparts];
      int[] varno = new int[nparts];
      int count = 0;
      for (PartitionForVariable2D part : partList) {
        partno[count] = part.partno;
        groupno[count] = part.groupno;
        varno[count] = part.varno;
        count++;
      }
      this.partnoSA =  new SmartArrayInt(partno);
      this.groupnoSA =  new SmartArrayInt(groupno);
      this.varnoSA =  new SmartArrayInt(varno);

      partList = null; // GC
    }

    // only used by PartitionBuilder, not PartitionBuilderFromIndex
    void addPartition(int partno, int groupno, int varno, int ndups, int nrecords, int nmissing,
        GribCollectionMutable.VariableIndex vi) {
      if (partList == null) partList = new ArrayList<>(nparts);
      partList.add(new PartitionForVariable2D(partno, groupno, varno));
      this.ndups += ndups;
      this.nrecords += nrecords;
      this.nmissing += nmissing;
    }

    @Override
    public String toStringComplete() {
      Formatter sb = new Formatter();
      sb.format("VariableIndexPartitioned%n");
      sb.format(" partno=");
      this.partnoSA.show(sb);
      sb.format("%n groupno=");
      this.groupnoSA.show(sb);
      sb.format("%n varno=");
      this.varnoSA.show(sb);
      sb.format("%n");
      int count = 0;
      sb.format("     %7s %3s %3s %6s %3s%n", "N", "dups", "Miss", "density", "partition");
      for (int i=0; i<nparts; i++) {
        int partWant = this.partnoSA.get(i);
        Partition part = partitions.get(partWant);
        sb.format("   %2d: %7d %s%n", count++, partWant, part.getFilename());
      }
      sb.format("%n");

      sb.format(super.toStringComplete());
      return sb.toString();
    }

  }

  //////////////////////////////////////////////////////////////////////////////////////////

  // wrapper around a GribCollection
  public class Partition implements Comparable<Partition> {
    final String name, directory;
    final String filename;
    long lastModified, fileSize;
    CalendarDate partitionDate;

    // temporary storage while building - do not use - must call getGribCollection()()
    // GribCollection gc;

    // constructor from ncx
    public Partition(String name, String filename, long lastModified, long fileSize, String directory, CalendarDate partitionDate) {
      this.name = name;
      this.filename = filename; // grib collection ncx
      this.lastModified = lastModified;
      this.fileSize = fileSize;
      this.directory = directory;
      this.partitionDate = partitionDate;
    }

    public String getName() {
      return name;
    }

    public String getFilename() {
      return filename;
    }

    public String getDirectory() {
      return directory;
    }

    public long getLastModified() {
      return lastModified;
    }

    public boolean isGrib1() {
      return isGrib1;         // in GribCollection
    }

    public FeatureCollectionConfig getConfig() {
      return config;   // in GribCollection
    }

    public org.slf4j.Logger getLogger() {
      return logger;          // in TimePartition
    }

    // null if it came from the index
    public MCollection getDcm() {
      return dcm;            // in GribCollection
    }

    private DateExtractor extractor;
    private DateExtractor getDateExtractor() {
      if (extractor == null)
         extractor = config.getDateExtractor();
      return extractor;
    }

    @Nullable
    String getIndexFilenameInCache() {
      File file = new File(directory, filename);
      File existingFile = GribIndexCache.getExistingFileOrCache(file.getPath());
      if (existingFile == null) {
        // try reletive to index file
        File parent = getIndexParentFile();
        if (parent == null) return null;
        existingFile = new File(parent, filename);
        if (!existingFile.exists()) return null;
      }
      return existingFile.getPath();
    }

    // acquire or construct GribCollection - caller must call gc.close() when done
    public GribCollectionImmutable getGribCollection() throws IOException {
      String path = getIndexFilenameInCache();
      if (path == null) {
        if (Grib.debugIndexOnly) {  // we are running in debug mode where we only have the indices, not the data files
          // tricky: substitute the current root
          File orgParentDir = new File(directory);
          File currentFile = new File(PartitionCollectionMutable.this.indexFilename);
          File currentParent = currentFile.getParentFile();
          File currentParentWithDir = new File(currentParent, orgParentDir.getName());
          File nestedIndex = isPartitionOfPartitions ? new File(currentParentWithDir, filename) : new File(currentParent, filename); // JMJ
          path = nestedIndex.getPath();
        } else {
          throw new FileNotFoundException("No index filename for partition= "+this.toString());
        }
      }

      // LOOK not cached
      return (GribCollectionImmutable) PartitionCollectionImmutable.partitionCollectionFactory.open(new DatasetUrl(null, path), -1, null, this);
    }

    // the children must already exist
    @Nullable
    public GribCollectionMutable makeGribCollection() {
      GribCollectionMutable result = GribCdmIndex.openMutableGCFromIndex(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), config, false, true, logger);
      if (result == null) {
        logger.error("Failed on openMutableGCFromIndex {}", dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX));
        return null;
      }
      lastModified = result.lastModified;
      fileSize = result.fileSize;
      if (result.masterRuntime != null)
        partitionDate = result.masterRuntime.getFirstDate();
      return result;
    }

    @Override
    public int compareTo(@Nonnull Partition o) {
      if (partitionDate != null && o.partitionDate != null)
        return partitionDate.compareTo(o.partitionDate);
      return name.compareTo(o.name);
    }

    @Override
    public String toString() {
      return "Partition{" +
              "dcm=" + dcm +
              ", name='" + name + '\'' +
              ", directory='" + directory + '\'' +
              ", filename='" + filename + '\'' +
              ", partitionDate='" + partitionDate + '\'' +
              ", lastModified='" + CalendarDate.of(lastModified) + '\'' +
              '}';
    }

    /////////////////////////////////////////////
    // only used during creation of index
    private MCollection dcm;

    // constructor from a MCollection object
    public Partition(MCollection dcm) {
      FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG);
      if (config == null)
        logger.warn("Partition missing FeatureCollectionConfig {}", dcm);

      this.dcm = dcm;
      this.name = dcm.getCollectionName();
      this.lastModified = dcm.getLastModified();
      this.directory = StringUtil2.replace(dcm.getRoot(), '\\', "/");
      this.partitionDate = dcm.getPartitionDate();

      String indexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/");
      if (partitionDate == null) {
        partitionDate = getDateExtractor().getCalendarDateFromPath(indexFilename);  // LOOK dicey
      }

      // now remove the directory
      if (indexFilename.startsWith(directory)) {
        indexFilename = indexFilename.substring(directory.length());
        if (indexFilename.startsWith("/")) indexFilename = indexFilename.substring(1);
      }
      filename = indexFilename;
    }

  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  protected final org.slf4j.Logger logger;
  protected List<Partition> partitions;
  protected boolean isPartitionOfPartitions;
  int[] run2part;   // masterRuntime.length; which partition to use for masterRuntime i

  protected PartitionCollectionMutable(String name, File directory, FeatureCollectionConfig config, boolean isGrib1, org.slf4j.Logger logger) {
    super(name, directory, config, isGrib1);
    this.logger = logger;
    this.partitions = new ArrayList<>();
    this.datasets = new ArrayList<>();
  }

  /**
   * Use partition names as the file names
   */
  @Override
  public List<String> getFilenames() {
    List<String> result = new ArrayList<>();
    for (Partition p : getPartitions()) {
      result.add(p.getIndexFilenameInCache());
    }
    Collections.sort(result);
    return result;
  }

  Partition addPartition(String name, String filename, long lastModified, long length,
      String directory, CalendarDate partitionDate) {
    Partition partition = new Partition(name, filename, lastModified, length, directory, partitionDate);
    partitions.add(partition);
    return partition;
  }

  public void addPartition(MCollection dcm) {
    Partition partition = new Partition(dcm);
    try (GribCollectionMutable gc = partition.makeGribCollection()) {  // make sure we can open the collection
      if (gc == null)
        logger.warn("failed to open partition {} =skipping", dcm.getCollectionName());
      else
        partitions.add(partition);
    } catch (Exception e) {
      logger.warn("failed to open partition {} -skipping", dcm.getCollectionName(), e);
    }
  }

  void sortPartitions() {
    Collections.sort(partitions);
    partitions = Collections.unmodifiableList(partitions);
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////

  // construct - going to write index

  /**
   * Create a VariableIndexPartitioned, add it to the given group
   *
   * @param group  the new VariableIndexPartitioned is in this group
   * @param from   copy info from here
   * @param nparts size of partition list
   * @return a new VariableIndexPartitioned
   */
  VariableIndexPartitioned makeVariableIndexPartitioned(GroupGC group,
      GribCollectionMutable.VariableIndex from, int nparts) {
    VariableIndexPartitioned vip = new VariableIndexPartitioned(group, from, nparts);
    group.addVariable(vip);

    if (from instanceof VariableIndexPartitioned && !isPartitionOfPartitions) {    // LOOK dont really understand this
      VariableIndexPartitioned vipFrom = (VariableIndexPartitioned) from;
      assert vipFrom.partList == null; // // check if vipFrom has been finished
      for (int i=0; i<vipFrom.nparts; i++)
        vip.addPartition(vipFrom.partnoSA.get(i), vipFrom.groupnoSA.get(i), vipFrom.varnoSA.get(i), 0, 0, 0, vipFrom);
    }

    return vip;
  }

  public Iterable<Partition> getPartitions() {
    return partitions;
  }

  Partition getPartition(int idx) {
    return partitions.get(idx);
  }

  int getPartitionSize() {
     return partitions.size();
   }

  public void showIndex(Formatter f) {
    super.showIndex(f);

    int count = 0;
    f.format("isPartitionOfPartitions=%s%n", isPartitionOfPartitions);
    f.format("Partitions%n");
    for (Partition p :  getPartitions())
      f.format("%d:  %s%n", count++, p);
    f.format("%n");

    if (run2part == null) f.format("run2part null%n");
    else {
      f.format(" master runtime -> partition %n");
      for (int idx=0; idx<masterRuntime.getSize(); idx++) {
        int partno =  run2part[idx];
        Partition part = getPartition(partno);
        f.format(" %d:  %s -> part %3d %s%n", count, masterRuntime.getRuntimeDate(idx), partno,  part);
        count++;
      }
      f.format("%n");
    }
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id:RegExpAndDurationTimeCoverageEnhancer.java 63 2006-07-12 21:50:51Z edavis $
package thredds.cataloggen.datasetenhancer;

import thredds.cataloggen.DatasetEnhancer;
import thredds.catalog.InvDataset;
import thredds.catalog.InvDatasetImpl;
import thredds.crawlabledataset.CrawlableDataset;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;

/**
 * Try to add timeCoverage to the InvDataset using a regular expression match,
 * capturing group replacement, and a duration string.
 *

 *
 Look for a match to the given matchPattern in the CrawlableDataset name.

 *
 Substitute all replacment patterns ("$n") in the substitionPattern with the capturing groups from the regular expression match.

 *
 Use the string obtained from the substitution as the start time and the given duration string to form a timeCoverage element for the InvDataset.

 *

 *
 * @author edavis
 * @since Dec 6, 2005 10:59:21 AM
 */
public class RegExpAndDurationTimeCoverageEnhancer implements DatasetEnhancer
{
  static private org.slf4j.Logger log =
          org.slf4j.LoggerFactory.getLogger( RegExpAndDurationTimeCoverageEnhancer.class );

  private final String matchPattern;
  private final String substitutionPattern;
  private final String duration;

  private final MatchTarget matchTarget;
  private enum MatchTarget
  {
    DATASET_NAME,
    DATASET_PATH
  }

  private java.util.regex.Pattern pattern;

  /**
   * Factory method that returns a RegExpAndDurationTimeCoverageEnhancer instance
   * that will apply the match pattern to the dataset name.
   *
   * @param matchPattern a regular expression used to match against the CrawlableDataset name.
   * @param substitutionPattern the time coverage start time (which may contain regular expression capturing group substitution strings).
   * @param duration the time coverage duration string.
   *
   * @return a RegExpAndDurationTimeCoverageEnhancer that will apply the match pattern to the dataset name.
   */
  public static
  RegExpAndDurationTimeCoverageEnhancer
  getInstanceToMatchOnDatasetName( String matchPattern,
                                   String substitutionPattern,
                                   String duration )
  {
    return new RegExpAndDurationTimeCoverageEnhancer(
            matchPattern, substitutionPattern,
            duration, MatchTarget.DATASET_NAME );
  }

  /**
   * Factory method that returns a RegExpAndDurationTimeCoverageEnhancer instance
   * that will apply the match pattern to the dataset path.
   *
   * @param matchPattern a regular expression used to match against the CrawlableDataset path.
   * @param substitutionPattern the time coverage start time (which may contain regular expression capturing group substitution strings).
   * @param duration the time coverage duration string.
   *
   * @return a RegExpAndDurationTimeCoverageEnhancer that will apply the match pattern to the dataset path.
   */
  public static RegExpAndDurationTimeCoverageEnhancer
  getInstanceToMatchOnDatasetPath( String matchPattern,
                                   String substitutionPattern,
                                   String duration )
  {
    return new RegExpAndDurationTimeCoverageEnhancer(
            matchPattern, substitutionPattern,
            duration, MatchTarget.DATASET_PATH );
  }

  /**
   * Constructor
   *
   * @param matchPattern a regular expression used to match against the CrawlableDataset name.
   * @param substitutionPattern used, after substitution with the regular expression capturing groups, as the start time for a time coverage.
   * @param duration used as the duration string for a time coverage.
   * @param matchTarget indicates what information to match on (e.g., the dataset path).
   */
  private RegExpAndDurationTimeCoverageEnhancer( String matchPattern,
                                                String substitutionPattern,
                                                String duration,
                                                MatchTarget matchTarget )
  {
    if ( matchPattern == null ) throw new IllegalArgumentException( "Null match pattern not allowed.");
    if ( substitutionPattern == null ) throw new IllegalArgumentException( "Null substitution pattern not allowed.");
    if ( duration == null ) throw new IllegalArgumentException( "Null duration not allowed.");
    if ( matchTarget == null ) throw new IllegalArgumentException( "Null match target not allowed.");

    this.matchPattern = matchPattern;
    this.substitutionPattern = substitutionPattern;
    this.duration = duration;
    this.matchTarget = matchTarget;

    try
    {
      this.pattern = java.util.regex.Pattern.compile( matchPattern );
    }
    catch ( java.util.regex.PatternSyntaxException e )
    {
      log.error( "ctor(): bad match pattern <" + this.matchPattern + ">, failed to compile: " +e.getMessage() );
      this.pattern = null;
    }
  }

  public MatchTarget getMatchTarget() { return this.matchTarget; }
  public String getMatchPattern() { return matchPattern; }
  public String getSubstitutionPattern() { return substitutionPattern; }
  public String getDuration() { return duration; }

  public Object getConfigObject() { return null; }

  public boolean addMetadata( InvDataset dataset, CrawlableDataset crDataset )
  {
    if ( this.pattern == null )
    {
      log.error( "addMetadata(): bad match pattern <" + this.matchPattern + ">." );
      return false;
    }

    String matchTargetString;
    if ( this.matchTarget.equals( MatchTarget.DATASET_NAME ))
      matchTargetString = crDataset.getName();
    else if ( this.matchTarget.equals( MatchTarget.DATASET_PATH ))
      matchTargetString = crDataset.getPath();
    else
      throw new IllegalStateException( "Unknown match target [" + this.matchTarget.toString() + "].");

    java.util.regex.Matcher matcher = this.pattern.matcher( matchTargetString );
    if ( ! matcher.find() )
    {
      return ( false ); // Pattern not found.
    }
    StringBuffer startTime = new StringBuffer();
    try
    {
      matcher.appendReplacement( startTime, this.substitutionPattern );
    }
    catch ( IndexOutOfBoundsException e )
    {
      log.error( "addMetadata(): capture group mismatch between match pattern <" + this.matchPattern + "> and substitution pattern <" + this.substitutionPattern + ">: " + e.getMessage());
      return( false);
    }
    startTime.delete( 0, matcher.start());

    try
    {
      ( (InvDatasetImpl) dataset ).setTimeCoverage(
              new DateRange( new DateType( startTime.toString(), null, null ), null,
                             new TimeDuration( this.duration ), null ) );
    }
    catch ( Exception e )
    {
      log.warn( "addMetadata(): Start time <" + startTime.toString() + "> or duration <" + this.duration + "> not parsable" +
                " (crDataset.getName() <" + crDataset.getName() + ">, this.matchPattern() <" + this.matchPattern + ">, this.substitutionPattern() <" + this.substitutionPattern + ">): " + e.getMessage() );
      return ( false );
    }

    return ( true );
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

// $Id: DqcFactory.java 48 2006-07-12 16:15:40Z caron $

package thredds.catalog.query;

import thredds.catalog.XMLEntityResolver;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.IO;

import org.jdom2.input.SAXBuilder;
import org.jdom2.*;

import java.io.*;
import java.net.*;
import java.util.*;

import ucar.nc2.util.DiskCache2;

/**
 * Reads an XML document and constructs an QueryCapability object.
 *

 *
Example of normal use:

 *

 *

 * DqcFactory fac = new DqcFactory(true);
 * QueryCapability dqc = fac.readXML(url);
 * System.out.println(" dqc hasFatalError= "+dqc.hasFatalError());
 * System.out.println(" dqc messages= \n"+dqc.getErrorMessages());
 * fac.writeXML(dqc, System.out);
 *

 *

 * Implementation details: Uses JAXP to load an XML Parser and construct a DOM tree.
 * Uses a pluggable "converter" to transform the DOM to the thredds.catalog.query objects.
 *
 * @author John Caron
 */

public class DqcFactory {
  public static boolean debugVersion = false;
  public static boolean showParsedXML = false;

  static private DiskCache2 diskCache = null;
  static private int buffer_size = 64000;

  static public void setPersistenceCache(DiskCache2 dc) {
    diskCache = dc;
  }
  ////////////////////////////////////////////////////////////////////////////////////////////////////

  private SAXBuilder builder; // JAXP parser
  private DqcConvertIF defaultConverter;

  private HashMap<String, String> versionToNamespaceHash = new HashMap<>(10);
  private HashMap<String, DqcConvertIF> namespaceToDqcConverterHash = new HashMap<>(10);
  private StringBuilder warnMessages, errMessages, fatalMessages;

  /**
   * Constructor.
   * Can use this to read as many catalogs as you want, but should only
   * use in single thread.
   *
   * @param validate : do XML validation or not.
   */
  public DqcFactory(boolean validate) {
    XMLEntityResolver jaxp = new XMLEntityResolver(validate);
    builder = jaxp.getSAXBuilder();
    warnMessages = jaxp.getWarningMessages();
    errMessages = jaxp.getErrorMessages();
    fatalMessages = jaxp.getFatalMessages();
    setDefaults();
  }

  private void setDefaults() {
    try {
      /* Class fac2 = Class.forName("thredds.catalog.parser.jdom.DqcConvert2");
      Object fac2o = fac2.newInstance();
      registerConverter( "0.2", XMLEntityResolver.DQC_NAMESPACE_02, (DqcConvertIF) fac2o);

      Class fac3 = Class.forName("thredds.catalog.parser.jdom.DqcConvert3");
      Object fac3o = fac3.newInstance();
      registerConverter( "0.3", XMLEntityResolver.DQC_NAMESPACE_03, (DqcConvertIF) fac3o); */

      Class fac4 = Class.forName("thredds.catalog.parser.jdom.DqcConvert4");
      Object fac4o = fac4.newInstance();
      defaultConverter = (DqcConvertIF) fac4o;
      registerConverter( "0.4", XMLEntityResolver.DQC_NAMESPACE_04, (DqcConvertIF) fac4o);

    } catch (ClassNotFoundException e) {
      throw new RuntimeException("DqcFactory: no implementing class found: " + e.getMessage());
    } catch (InstantiationException e) {
      throw new RuntimeException("DqcFactory: instantiation failed: " + e.getMessage());
    } catch (IllegalAccessException e) {
      throw new RuntimeException("DqcFactory: access failed: " + e.getMessage());
    }
  }

  private void registerConverter(String version, String namespace, DqcConvertIF converter) {
    namespaceToDqcConverterHash.put(namespace, converter);
    versionToNamespaceHash.put(version, namespace);
    // converter.setCatalogFactory( this);
  }

  public void appendErr(String err) {
    errMessages.append(err);
  }

  public void appendFatalErr(String err) {
    fatalMessages.append(err);
  }

  public void appendWarning(String err) {
    warnMessages.append(err);
  }

  public QueryCapability readXML( String docAsString, URI uri ) throws IOException
  {
    // get ready for XML parsing
    warnMessages.setLength( 0 );
    errMessages.setLength( 0 );
    fatalMessages.setLength( 0 );

    Document doc = null;
    try
    {
      doc = builder.build( new StringReader( docAsString ) );
    }
    catch ( JDOMException e )
    {
      fatalMessages.append( e.getMessage() ); // makes it invalid
    }

    return readXML( doc, uri );
  }

  /**
   * Create an QueryCapability from an XML document at a named URL.
   * check dqc.isValid, dqc.getErrorMessages() to see if ok.
   * If Disk caching is set, cache the dqc and check IfModifiedSince.
   *
   * @param uriString : the URI name that the XML doc is at.
   * @return an QueryCapability object
   * @throws IOException on failure
   */
  public QueryCapability readXML(String uriString) throws IOException {

    // get URI
    URI uri;
    try {
      uri = new URI(uriString);
    } catch (URISyntaxException e) {
      throw new MalformedURLException(e.getMessage());
    }

    // check if its cached
    if (diskCache != null) {
      File file = diskCache.getCacheFile(uriString);
      if (file != null) {
        HttpURLConnection conn = null;

        try {
          URL url = uri.toURL();
          conn = (HttpURLConnection) url.openConnection();
          conn.setRequestMethod("GET");
          conn.setIfModifiedSince(file.lastModified());

          int code = conn.getResponseCode();
          if (code == HttpURLConnection.HTTP_OK) {
            java.io.InputStream is = conn.getInputStream();
            if (is != null) {
              try (FileOutputStream fout = new FileOutputStream(file)) {
                  IO.copyB(is, fout, buffer_size);  // cache it
              }
              try (InputStream fin = new BufferedInputStream( new FileInputStream(file), 50000)) {
                return readXML(fin, uri);
              }
            }

          } else {
            // use file
            try (FileInputStream fin = new FileInputStream(file)) {
              return readXML(fin, uri);
            }
          }

        } finally {

          if (conn != null)
            conn.disconnect();
        }

      } // has file

      // no file - read and cache
      IO.readURLtoFileWithExceptions(uriString, file, buffer_size);
      try (InputStream fin = new BufferedInputStream(
              new FileInputStream(file), 50000)) {
        return readXML(fin, uri);
      }

    } // has diskCache

    // otherwise just open the URL
    warnMessages.setLength(0);
    errMessages.setLength(0);
    fatalMessages.setLength(0);

    Document doc = null;
    try {
      doc = builder.build(uriString);
    } catch (JDOMException e) {
      fatalMessages.append(e.getMessage()); // makes it invalid
    }

    return readXML(doc, uri);
  }

  /**
   * Create an QueryCapability from an InputStream.
   * check dqc.isValid, dqc.getErrorMessages() to see if ok.
   *
   * @param docIs : the InputStream to read from
   * @param uri   : the URI of the document, used for resolving reletive references.
   * @return an QueryCapability object
   * @throws IOException on failure
   */
  public QueryCapability readXML(InputStream docIs, URI uri) throws IOException {

    // get ready for XML parsing
    warnMessages.setLength(0);
    errMessages.setLength(0);
    fatalMessages.setLength(0);

    Document doc = null;
    try {
      doc = builder.build(docIs);
    } catch (JDOMException e) {
      fatalMessages.append(e.getMessage()); // makes it invalid
    }

    return readXML(doc, uri);
  }

  /**
   * Create an InvCatalog from an a JDOM document.
   * check dqc.isValid, dqc.getErrorMessages() to see if ok.
   *
   * @param doc parse this document
   * @param uri : the URI of the document, used for resolving reletive references.
   * @return an InvCatalogImpl object
   * @throws IOException on failure
   */
  private QueryCapability readXML(org.jdom2.Document doc, URI uri) throws IOException {

    if (doc == null) { // parse failed
      QueryCapability dqc = new QueryCapability();
      if (fatalMessages.length() > 0)
        dqc.appendErrorMessage(fatalMessages.toString(), true); // makes it invalid
      if (errMessages.length() > 0)
        dqc.appendErrorMessage(errMessages.toString(), false); // doesnt make it invalid
      if (errMessages.length() > 0)
        dqc.appendErrorMessage(warnMessages.toString(), false); // doesnt make it invalid
      return dqc;
    }

    // decide on converter based on namespace
    Element root = doc.getRootElement();
    String namespace = root.getNamespaceURI();
    DqcConvertIF fac = namespaceToDqcConverterHash.get(namespace);
    if (fac == null) {
      fac = defaultConverter; // LOOK
      if (debugVersion)
        System.out.println("use default converter " + fac.getClass().getName() + "; no namespace " + namespace);
    } else if (debugVersion)
      System.out.println("use converter " + fac.getClass().getName() + " based on namespace " + namespace);

// convert to object model
    QueryCapability dqc = fac.parseXML(this, doc, uri);
    if (fatalMessages.length() > 0)
      dqc.appendErrorMessage(fatalMessages.toString(), true); // makes it invalid
    if (errMessages.length() > 0)
      dqc.appendErrorMessage(errMessages.toString(), false); // doesnt make it invalid
    if (errMessages.length() > 0)
      dqc.appendErrorMessage(warnMessages.toString(), false); // doesnt make it invalid
    return dqc;
  }

  /**
   * Write the catalog as an XML document to a String.
   *
   * @param dqc : write this QueryCapability to an XML representation.
   * @return string containing XML representation
   * @throws IOException on failure
   */
  public String writeXML(QueryCapability dqc) throws IOException {
    ByteArrayOutputStream os = new ByteArrayOutputStream(10000);
    writeXML(dqc, os);
    return os.toString(CDM.utf8Charset.name());
  }

  /**
   * Write the catalog as an XML document to the specified stream.
   *
   * @param dqc : write this QueryCapability to an XML representation.
   * @param os  write to this OutputStream
   * @throws IOException on an error.
   */
  public void writeXML(QueryCapability dqc, OutputStream os) throws IOException {
    String ns = versionToNamespaceHash.get( dqc.getVersion() );
    DqcConvertIF fac = namespaceToDqcConverterHash.get( ns );

    if ( fac == null )
      fac = defaultConverter;

    fac.writeXML( dqc, os );
  }

  /**
   * Write the catalog as an XML document to the specified filename.
   *
   * @param dqc      : write this QueryCapability to an XML representation.
   * @param filename write to this filename
   * @return true if success
   */
  public boolean writeXML(QueryCapability dqc, String filename) {
    try {
      BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(filename));
      writeXML(dqc, os);
      os.close();
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    }
    return true;
  }

/************************************************************************/
  /**
   * testing
   */

  private static void doOne(DqcFactory fac, String url) {
    System.out.println("***read " + url);
    try {
      QueryCapability dqc = fac.readXML(url);
      System.out.println(" dqc hasFatalError= " + dqc.hasFatalError());
      System.out.println(" dqc messages= \n" + dqc.getErrorMessages());
      fac.writeXML(dqc, System.out);
    } catch (Exception e) {
      e.printStackTrace();
    }

  }

  public static void main(String[] args) throws Exception {
    DqcFactory fac = new DqcFactory(true);

    doOne(fac, "file:///C:/data/dqc/metarDQC.xml");
  }

}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import java.io.*;
import java.util.Formatter;

/**
 * This class holds information about each dimension in a DArray.
 * Each array dimension carries with it its own projection information, as
 * well as it's name and size.
 *
 The projection information takes the form of three integers: the start,
 * stop, and stride values. This is clearest with an example. Consider a
 * one-dimensional array 10 elements long. If the start value of the
 * dimension constraint is 3, then the constrained array appears to be seven
 * elements long. If the stop value is changed to 7, then the array appears
 * to be five elements long. If the stride is changed to two, the array will
 * appear to be 3 elements long. Array constraints are written as
 * [start:stride:stop].
 *
 * @see DArray
 */
public final class DArrayDimension extends DAPNode
{
    static final long serialVersionUID = DArrayDimension.class.hashCode();

    static public class Slice {
        public int size;
        public int start;
        public int stride;
        public int stop;
        public Slice(int size, int start, int stride, int stop)
        {
            this.size = size;
            this.start = start;
            this.stride = stride;
            this.stop = stop;
        }
        public Slice(int start, int stride, int stop)
        {
            this(0,start,stride,stop);
            this.size = 1 + (stop - start) / stride;
        }
    };

    // Define two slices for each dimension
    transient private Slice decl = null; // as defined in the original declaration
    transient private Slice projection = null; // as defined in any projection of this dimension
    private DArray container = null;

    /**
     * Construct a new DArrayDimension.
     *
     * @param size The size of the dimension.
     * @param name The dimension's name, or null if no name.
     */
    public DArrayDimension(int size, String name) {
        super(name);
        this.decl = new Slice(size,0,1,size-1);
    }

    /**
     * Get the dimension size.
     */
    public int getSize() {
        return (projection != null?projection.size:decl.size);
    }

    /**
     * Set the dimension size.
     */
    public void setSize(int size) {
        decl.size = size;
    }

    /**
     * Get the projection start point for this dimension.
     */
    public int getStart() {
        return (projection != null?projection.start:decl.start);
    }

    /**
     * Get the projection stride size for this dimension.
     */
    public int getStride() {
        return (projection != null?projection.stride:decl.stride);
    }

    /**
     * Get the projection stop point for this dimension.
     */
    public int getStop() {
        return (projection != null?projection.stop:decl.stop);
    }

    public void setContainer(DArray da)
    {
        container = da;
    }

    public DArray getContainer() {return container;}

    public void printConstraint(PrintWriter os)
    {
	String buf = "[";
	if(getSize() == 1 && getStart() == getStop())
	    buf += ""+getStart();
	else if(getStride() == 1)
	    buf += getStart() + ":" + getStop();
	else
	    buf += getStart() + ":" + getStride() + ":" + getStop();
	buf += "]";
	os.print(buf);
    }

    /**
     * Set the projection information for this dimension.
     * The parameters start stride and stop
     * are checked to verify that they make sense relative to each other and to
     * the size of this dimension. If not an Invalid ParameterException is thrown.
     * The general rule is: 0<=start<size, 0<stride, 0<=stop<size, start<=stop.
     *
     * @param start  The starting point for the projection of this DArrayDimension.
     * @param stride The size of the stride for the projection of this DArrayDimension.
     * @param stop   The stopping point for the projection of this DArrayDimension.
     */
    public void setProjection(int start, int stride, int stop) throws InvalidDimensionException
    {
        if(projection != null) {
            // See if we are changing start/stride/stop
          if(getSize() != start || getStride() != stride || getStop() != stop) {
            Formatter f = new Formatter();
            f.format(" [%d:%d:%d (%d)] != [%d:%d:%d (%d)]", start, stride, stop, projection.size, projection.start, projection.stride, projection.stop, projection.size);
            throw new ConstraintException("Implementation limitation: muliple references to same variable in single constraint: "+container.getLongName()+f.toString());
          }
        }

        String msg = "DArrayDimension.setProjection: Bad Projection Request: ";

        // validate the arguments

        if (start < 0)
            throw new InvalidDimensionException(msg + "start < 0");

        if (stride <= 0)
            throw new InvalidDimensionException(msg + "stride <= 0");

        if (stop < 0)
             throw new InvalidDimensionException(msg + "stop < 0");

        if (start < decl.start)
                   throw new InvalidDimensionException(msg + "start (" + start + ") < size (" + decl.size + ") for " + _nameClear);

        if (stop >= decl.size)
            throw new InvalidDimensionException(msg + "stop >= size: "+stop+":"+decl.size);

        if (stop < start)
            throw new InvalidDimensionException(msg + "stop < start");

        projection = new Slice(start,stride,stop);
    }

    /**
     * Returns a clone of this Array.
     * See DAPNode.cloneDag()
     *
     * @param map track previously cloned nodes
     * @return a clone of this object.
     */
    public DAPNode cloneDAG(CloneMap map)
        throws CloneNotSupportedException
    {
        DArrayDimension d = (DArrayDimension) super.cloneDAG(map);
        if(container != null) d.container = (DArray)cloneDAG(map,container);
        return d;
    }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.hdf4;

import ucar.nc2.constants.DataFormatType;
import ucar.nc2.iosp.*;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.Structure;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.IO;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.io.PositioningDataInputStream;
import ucar.ma2.*;

import java.io.IOException;
import java.io.InputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.List;
import java.nio.ByteBuffer;

/**
 * @author caron
 * @since Dec 17, 2007
 */
public class H4iosp extends AbstractIOServiceProvider {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H4iosp.class);
  static private boolean showLayoutTypes = false;

  private H4header header = new H4header();

  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    return H4header.isValidFile(raf);
  }

    public String getFileTypeId() {
      if (header.isEos()) return "HDF4-EOS";
      return DataFormatType.HDF4.getDescription();
    }

    public String getFileTypeDescription() {
      return "Hierarchical Data Format, version 4";
    }

  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    super.open(raf, ncfile, cancelTask);
    header.read(raf, ncfile);
    ncfile.finish();
  }

  public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
    if (v instanceof Structure)
      return readStructureData((Structure) v, section);

    H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
    DataType dataType = v.getDataType();
    vinfo.setLayoutInfo();   // make sure needed info is present

    // make sure section is complete
    section = Section.fill(section, v.getShape());

    if (vinfo.hasNoData) {
      Object arr = (vinfo.fillValue == null) ? IospHelper.makePrimitiveArray((int) section.computeSize(), dataType) :
        IospHelper.makePrimitiveArray((int) section.computeSize(), dataType, vinfo.fillValue);
      if (dataType == DataType.CHAR)
        arr = IospHelper.convertByteToChar((byte[]) arr);
      return Array.factory(dataType, section.getShape(), arr);
    }

    if (!vinfo.isCompressed) {
      if (!vinfo.isLinked && !vinfo.isChunked) {
        Layout layout = new LayoutRegular(vinfo.start, v.getElementSize(), v.getShape(), section);
        Object data = IospHelper.readDataFill(raf, layout, dataType, vinfo.fillValue, -1);
        return Array.factory(dataType, section.getShape(), data);

      } else if (vinfo.isLinked) {
        Layout layout = new LayoutSegmented(vinfo.segPos, vinfo.segSize, v.getElementSize(), v.getShape(), section);
        Object data = IospHelper.readDataFill(raf, layout, dataType, vinfo.fillValue, -1);
        return Array.factory(dataType, section.getShape(), data);

      } else if (vinfo.isChunked) {
        H4ChunkIterator chunkIterator = new H4ChunkIterator(vinfo);
        Layout layout = new LayoutTiled(chunkIterator, vinfo.chunkSize, v.getElementSize(), section);
        Object data = IospHelper.readDataFill(raf, layout, dataType, vinfo.fillValue, -1);
        return Array.factory(dataType, section.getShape(), data);
      }

    } else {
      if (!vinfo.isLinked && !vinfo.isChunked) {
        if (showLayoutTypes) System.out.println("***notLinked, compressed");
        Layout index = new LayoutRegular(0, v.getElementSize(), v.getShape(), section);
        InputStream is = getCompressedInputStream(vinfo);
        PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
        Object data = IospHelper.readDataFill(dataSource, index, dataType, vinfo.fillValue);
        return Array.factory(dataType, section.getShape(), data);

      } else if (vinfo.isLinked) {
        if (showLayoutTypes) System.out.println("***Linked, compressed");
        Layout index = new LayoutRegular(0, v.getElementSize(), v.getShape(), section);
        InputStream is = getLinkedCompressedInputStream(vinfo);
        PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
        Object data = IospHelper.readDataFill(dataSource, index, dataType, vinfo.fillValue);
        return Array.factory(dataType, section.getShape(), data);

      } else if (vinfo.isChunked) {
        LayoutBBTiled.DataChunkIterator chunkIterator = new H4CompressedChunkIterator(vinfo);
        LayoutBB layout = new LayoutBBTiled(chunkIterator, vinfo.chunkSize, v.getElementSize(), section);
        Object data = IospHelper.readDataFill(layout, dataType, vinfo.fillValue);
        return Array.factory(dataType, section.getShape(), data);
      }
    }

    throw new IllegalStateException();
  }

  /**
   * Structures must be fixed sized
   *
   * @param s       the record structure
   * @param section the record range to read
   * @return an ArrayStructure, with all the data read in.
   * @throws IOException                    on error
   * @throws ucar.ma2.InvalidRangeException if invalid section
   */
  private ucar.ma2.ArrayStructure readStructureData(ucar.nc2.Structure s, Section section) throws java.io.IOException, InvalidRangeException {
    H4header.Vinfo vinfo = (H4header.Vinfo) s.getSPobject();
    vinfo.setLayoutInfo();   // make sure needed info is present
    int recsize = vinfo.elemSize;

    // create the ArrayStructure
    StructureMembers members = s.makeStructureMembers();
    for (StructureMembers.Member m : members.getMembers()) {
      Variable v2 = s.findVariable(m.getName());
      H4header.Minfo minfo = (H4header.Minfo) v2.getSPobject();
      m.setDataParam(minfo.offset);
    }
    members.setStructureSize(recsize);
    ArrayStructureBB structureArray = new ArrayStructureBB(members, section.getShape());  // LOOK subset

    // loop over records
    byte[] result = structureArray.getByteBuffer().array();

    /*if (vinfo.isChunked) {
      InputStream is = getChunkedInputStream(vinfo);
      PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
      Layout layout = new LayoutRegular(vinfo.start, recsize, s.getShape(), section);
      IospHelper.readData(dataSource, layout, DataType.STRUCTURE, result);  */

    if (!vinfo.isLinked && !vinfo.isCompressed) {
      Layout layout = new LayoutRegular(vinfo.start, recsize, s.getShape(), section);
      IospHelper.readData(raf, layout, DataType.STRUCTURE, result, -1, true);

      /* option 1
    } else if (vinfo.isLinked && !vinfo.isCompressed) {
      Layout layout = new LayoutSegmented(vinfo.segPos, vinfo.segSize, recsize, s.getShape(), section);
      IospHelper.readData(raf, layout, DataType.STRUCTURE, result, -1);  */

      // option 2
    } else if (vinfo.isLinked && !vinfo.isCompressed) {
      InputStream is = new LinkedInputStream(vinfo);
      PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
      Layout layout = new LayoutRegular(0, recsize, s.getShape(), section);
      IospHelper.readData(dataSource, layout, DataType.STRUCTURE, result);

    } else if (!vinfo.isLinked && vinfo.isCompressed) {
      InputStream is = getCompressedInputStream(vinfo);
      PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
      Layout layout = new LayoutRegular(0, recsize, s.getShape(), section);
      IospHelper.readData(dataSource, layout, DataType.STRUCTURE, result);

    } else if (vinfo.isLinked && vinfo.isCompressed) {
      InputStream is = getLinkedCompressedInputStream(vinfo);
      PositioningDataInputStream dataSource = new PositioningDataInputStream(is);
      Layout layout = new LayoutRegular(0, recsize, s.getShape(), section);
      IospHelper.readData(dataSource, layout, DataType.STRUCTURE, result);

    } else {
      throw new IllegalStateException();
    }

    return structureArray;
  }

  public String toStringDebug(Object o) {
    if (o instanceof Variable) {
      Variable v = (Variable) o;
      H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
      return (vinfo != null) ? vinfo.toString() : "";
    }
    return null;
  }

  private InputStream getCompressedInputStream(H4header.Vinfo vinfo) throws IOException {
    // probably could construct an input stream from a channel from a raf
    // for now, just read it in.
    //System.out.println(" read "+ vinfo.length+" from "+ vinfo.start);
    byte[] buffer = new byte[vinfo.length];
    raf.seek(vinfo.start);
    raf.readFully(buffer);
    ByteArrayInputStream in = new ByteArrayInputStream(buffer);
    return new java.util.zip.InflaterInputStream(in);
  }

  private InputStream getLinkedCompressedInputStream(H4header.Vinfo vinfo) {
    return new java.util.zip.InflaterInputStream(new LinkedInputStream(vinfo));
  }

  private class LinkedInputStream extends InputStream {
    byte[] buffer;

    int nsegs;
    long[] segPosA;
    int[] segSizeA;

    int segno = -1;
    int segpos = 0;
    int segSize = 0;
    //H4header.Vinfo vinfo;

    LinkedInputStream(H4header.Vinfo vinfo) {
      segPosA = vinfo.segPos;
      segSizeA = vinfo.segSize;
      nsegs = segSizeA.length;
    }

    LinkedInputStream(H4header.SpecialLinked linked) throws IOException {
      List<H4header.TagLinkedBlock>  linkedBlocks = linked.getLinkedDataBlocks();
      nsegs = linkedBlocks.size();
      segPosA = new long[nsegs];
      segSizeA = new int[nsegs];
      int count = 0;
      for (H4header.TagLinkedBlock tag : linkedBlocks) {
        segPosA[count] = tag.offset;
        segSizeA[count] = tag.length;
        count++;
      }
    }

    private boolean readSegment() throws IOException {
      segno++;
      if (segno == nsegs)
        return false;

      segSize = segSizeA[segno];
      while (segSize == 0) { // for some reason may have a 0 length segment
        segno++;
        if (segno == nsegs)
          return false;
        segSize = segSizeA[segno];
      }

      buffer = new byte[segSize];  // Look: could do this in buffer size 4096 to save memory
      raf.seek(segPosA[segno]);
      raf.readFully(buffer);
      segpos = 0;

      return true;
    }

    /*public int read(byte b[]) throws IOException {
      //System.out.println("request "+b.length);
      return super.read(b);
    }

    public int read(byte b[], int off, int len) throws IOException {
      //System.out.println("request "+len+" buffer size="+b.length+" offset="+off);
      return super.read(b, off, len);
    } */

    public int read() throws IOException {
      if (segpos == segSize) {
        boolean ok = readSegment();
        if (!ok) return -1;
      }

      int b = buffer[segpos] & 0xff;
      //System.out.println("  byte "+b+ " at= "+segpos);
      segpos++;
      return b;
    }
  }

  private InputStream getChunkedInputStream(H4header.Vinfo vinfo) {
    return new ChunkedInputStream(vinfo);
  }

  // look not implemented : skip data
  private class ChunkedInputStream extends InputStream {
    List<H4header.DataChunk> chunks;
    int chunkNo;

    byte[] buffer;
    int segPos, segSize;

    ChunkedInputStream(H4header.Vinfo vinfo) {
      this.chunks = vinfo.chunks;
      this.chunkNo = 0;
    }

    private void readChunk() throws IOException {
      H4header.DataChunk chunk = chunks.get(chunkNo);
      H4header.TagData chunkData = chunk.data;
      chunkNo++;

      if (chunkData.ext_type == TagEnum.SPECIAL_COMP) {
        // read compressed data in
        H4header.TagData cdata = chunkData.compress.getDataTag();
        byte[] cbuffer = new byte[cdata.length];
        raf.seek(cdata.offset);
        raf.readFully(cbuffer);
        // System.out.println("  read compress " + cdata.length + " at= " + cdata.offset);

        // uncompress it
        if (chunkData.compress.compress_type == TagEnum.COMP_CODE_DEFLATE) {
          InputStream in = new java.util.zip.InflaterInputStream(new ByteArrayInputStream(cbuffer));
          ByteArrayOutputStream out = new ByteArrayOutputStream(chunkData.compress.uncomp_length);
          IO.copy(in, out);
          buffer = out.toByteArray();
          //System.out.println("  uncompress "+buffer.length + ", wanted="+chunkData.compress.uncomp_length);
        } else {
          throw new IllegalStateException("unknown compression type =" + chunkData.compress.compress_type);
        }

      } else {
        buffer = new byte[chunkData.length];
        raf.seek(chunkData.offset);
        raf.readFully(buffer);
      }

      segPos = 0;
      segSize = buffer.length;
    }

    public int read() throws IOException {
      if (segPos == segSize)
        readChunk();
      int b = buffer[segPos] & 0xff;
      //System.out.println("  byte "+b+ " at= "+segPos);
      segPos++;
      return b;
    }

    /* public int read(byte b[]) throws IOException {
      System.out.println("request "+b.length);
      return super.read(b);
    }

    public int read(byte b[], int off, int len) throws IOException {
      System.out.println("request "+len+" buffer size="+b.length+" offset="+off);
      return super.read(b, off, len);
    } // */
  }

  private static class H4ChunkIterator implements LayoutTiled.DataChunkIterator {
    List<H4header.DataChunk> chunks;
    int chunkNo;

    H4ChunkIterator(H4header.Vinfo vinfo) {
      this.chunks = vinfo.chunks;
      this.chunkNo = 0;
    }

    public boolean hasNext() {
      return chunkNo < chunks.size();
    }

    public LayoutTiled.DataChunk next() throws IOException {
      H4header.DataChunk chunk = chunks.get(chunkNo);
      H4header.TagData chunkData = chunk.data;
      chunkNo++;

      return new LayoutTiled.DataChunk(chunk.origin, chunkData.offset);
    }
  }

  private class H4CompressedChunkIterator implements LayoutBBTiled.DataChunkIterator {
    List<H4header.DataChunk> chunks;
    int chunkNo;

    H4CompressedChunkIterator(H4header.Vinfo vinfo) {
      this.chunks = vinfo.chunks;
      this.chunkNo = 0;
    }

    public boolean hasNext() {
      return chunkNo < chunks.size();
    }

    public LayoutBBTiled.DataChunk next() throws IOException {
      H4header.DataChunk chunk = chunks.get(chunkNo);
      H4header.TagData chunkData = chunk.data;
      assert (chunkData.ext_type == TagEnum.SPECIAL_COMP);
      chunkNo++;

      return new DataChunk(chunk.origin, chunkData.compress);
    }
  }

  private class DataChunk implements LayoutBBTiled.DataChunk {
    private int[] offset; // offset index of this chunk, reletive to entire array
    private H4header.SpecialComp compress;
    private ByteBuffer bb;  // the data is placed into here

    DataChunk(int[] offset, H4header.SpecialComp compress) {
      this.offset = offset;
      this.compress = compress;
    }

    public int[] getOffset() {
      return offset;
    }

    public ByteBuffer getByteBuffer() throws IOException {
      if (bb == null) {
        // read compressed data in
        H4header.TagData cdata = compress.getDataTag();
        InputStream in;

        // compressed data stored in one place
        if (cdata.linked == null) {
          byte[] cbuffer = new byte[cdata.length];
          raf.seek(cdata.offset);
          raf.readFully(cbuffer);
          in = new ByteArrayInputStream(cbuffer);

        } else { // or compressed data stored in linked storage
          in = new LinkedInputStream(cdata.linked);
        }
        //System.out.println("  read compress " + cdata.length + " at= " + cdata.offset);

        // uncompress it
        if (compress.compress_type == TagEnum.COMP_CODE_DEFLATE) {
          // read the stream in and uncompress
          InputStream zin = new java.util.zip.InflaterInputStream(in);
          ByteArrayOutputStream out = new ByteArrayOutputStream(compress.uncomp_length);
          IO.copy(zin, out);
          byte[] buffer = out.toByteArray();
          bb = ByteBuffer.wrap(buffer);
          //System.out.println("  uncompress " + buffer.length + ", wanted=" + compress.uncomp_length);

        } else if (compress.compress_type == TagEnum.COMP_CODE_NONE) {
          // just read the stream in
          ByteArrayOutputStream out = new ByteArrayOutputStream(compress.uncomp_length);
          IO.copy(in, out);
          byte[] buffer = out.toByteArray();
          bb = ByteBuffer.wrap(buffer);
        } else {
          throw new IllegalStateException("unknown compression type =" + compress.compress_type);
        }
      }

      return bb;
    }

  }

  @Override
  public void reacquire() throws IOException {
    super.reacquire();
    header.raf = this.raf;
  }

  public Object sendIospMessage(Object message) {
    if (message.toString().equals("header"))
      return header;
    return super.sendIospMessage(message);
  }

}