/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.fmrc;

import ucar.nc2.time.CalendarDate;

import java.util.*;

/**
 * Inventory for a Forecast Model Run - one runtime.
 * Track inventory by coordinate value, not index.
 * Composed of one or more GridDatasets, each described by a GridDatasetInv.
 * For each Grid, the vert, time and ens coordinates are created as the union of the components.
 * We make sure we are sharing coordinates across grids where they are equivilent.
 * We are thus making a rectangular array var(time, ens, level).
 * So obviously we have to tolerate missing data.
 * <p/>
 * seems to be immutable after finish() is called.
 *
 * @author caron
 * @since Jan 11, 2010
 */
public class FmrInv implements Comparable<FmrInv> {
  //static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(FmrInv.class);

  private final List<TimeCoord> timeCoords = new ArrayList<>(); // list of unique TimeCoord
  private final List<EnsCoord> ensCoords = new ArrayList<>(); // list of unique EnsCoord
  private final List<VertCoord> vertCoords = new ArrayList<>(); // list of unique VertCoord
  private final Map<String, GridVariable> uvHash = new HashMap<>(); // hash of FmrInv.Grid
  private List<GridVariable> gridList;              // sorted list of FmrInv.Grid

  public List<TimeCoord> getTimeCoords() {
    return timeCoords;
  }

  public List<EnsCoord> getEnsCoords() {
    return ensCoords;
  }

  public List<VertCoord> getVertCoords() {
    return vertCoords;
  }

  public List<GridVariable> getGrids() {
    return gridList;
  }

  public List<GridDatasetInv> getInventoryList() {
    return invList;
  }

  public CalendarDate getRunDate() {
    return runtime;
  }

  public String getName() {
    return "";
  }

  ////////////////////////////////////////////////////////////////////////////////////

  private final List<GridDatasetInv> invList = new ArrayList<>();
  private final CalendarDate runtime;

  FmrInv(CalendarDate runtime) {
    this.runtime = runtime;
  }

  void addDataset(GridDatasetInv inv, Formatter debug) {
    invList.add(inv);

    if (debug != null) {
      debug.format(" Fmr add GridDatasetInv %s = ", inv.getLocation());
      for (TimeCoord tc : inv.getTimeCoords()) {
        debug.format("  %s %n", tc);
      }
    }

    // invert tc -> grid
    for (TimeCoord tc : inv.getTimeCoords()) {
      for (GridDatasetInv.Grid grid : tc.getGridInventory()) {
        GridVariable uv = uvHash.get(grid.getName());
        if (uv == null) {
          uv = new GridVariable(grid.getName());
          uvHash.put(grid.getName(), uv);
        }
        uv.addGridDatasetInv(grid);
      }
    }
  }

  // call after adding all runs
  void finish() {
    gridList = new ArrayList<>(uvHash.values());
    Collections.sort(gridList);

    // find the common coordinates
    for (GridVariable grid : gridList) {
      grid.finish();
    }

    // assign sequence number for time
    int seqno = 0;
    for (TimeCoord tc : timeCoords)
      tc.setId(seqno++);

    // assign sequence number for vertical coords with same name
    HashMap<String, List<VertCoord>> map = new HashMap<>();
    for (VertCoord vc : vertCoords) {
      List<VertCoord> list = map.get(vc.getName());
      if (list == null) {
        list = new ArrayList<>();
        map.put(vc.getName(), list);
      }
      list.add(vc);
    }
    for (List<VertCoord> list : map.values()) {
      if (list.size() > 0) {
        int count = 0;
        for (VertCoord vc : list) {
          if (count > 0) vc.setName(vc.getName()+count);
          count++;
        }
      }
    }

  }

  @Override
  public int compareTo(FmrInv fmr) {
    return runtime.compareTo(fmr.getRunDate());
  }

  /**
   * A grid variable for an fmr (one run)
   * A collection of GridDatasetInv.Grid, one for each seperate dataset. All have the same runDate.
   * The time and vert coord of the GridVariable is the union of the GridDatasetInv.Grid time and vert coords.
   *
   * @author caron
   * @since Jan 12, 2010
   */
  public class GridVariable implements Comparable {
    private final String name;
    private final List<GridDatasetInv.Grid> gridList = new ArrayList<>();
    VertCoord vertCoordUnion = null; // union of vert coords
    EnsCoord ensCoordUnion = null; // union of ens coords NOT USED YET
    TimeCoord timeCoordUnion = null; // union of time coords
    TimeCoord timeExpected = null; // expected time coords
    //private int countInv, countExpected;

    GridVariable(String name) {
      this.name = name;
    }

    public String getName() {
      return name;
    }

    public CalendarDate getRunDate( ) {
      return FmrInv.this.getRunDate();
    }

    void addGridDatasetInv(GridDatasetInv.Grid grid) {
      gridList.add(grid);
    }

    public List<GridDatasetInv.Grid> getInventory() { return gridList; }

    public TimeCoord getTimeExpected() { return timeExpected; }

    public TimeCoord getTimeCoord() { return timeCoordUnion; }

    public int compareTo(Object o) {
      GridVariable uv = (GridVariable) o;
      return name.compareTo(uv.name);
    }

    public int getNVerts() {
      return (vertCoordUnion == null) ? 1 : vertCoordUnion.getSize();
    }

    public int countTotal() {
      int total = 0;
      for (GridDatasetInv.Grid grid : gridList)
        total += grid.countTotal();
      return total;
    }

    void finish() {
      if (gridList.size() == 1) {
        GridDatasetInv.Grid grid = gridList.get(0);
        ensCoordUnion = EnsCoord.findEnsCoord(getEnsCoords(), grid.ec);
        vertCoordUnion = VertCoord.findVertCoord(getVertCoords(), grid.vc);
        timeCoordUnion = TimeCoord.findTimeCoord(getTimeCoords(), grid.tc);
        return;
      }

      // run over all ensCoords and construct the union
      List<EnsCoord> ensList = new ArrayList<>();
      EnsCoord ec_union = null;
      for (GridDatasetInv.Grid grid : gridList) {
        EnsCoord ec = grid.ec;
        if (ec == null) continue;
        if (ec_union == null)
          ec_union = new EnsCoord(ec);
        else if (!ec_union.equalsData(ec))
          ensList.add(ec);
      }
      if (ec_union != null) {
        if (ensList.size() > 0) EnsCoord.normalize(ec_union, ensList); // add the other coords
        ensCoordUnion = EnsCoord.findEnsCoord(getEnsCoords(), ec_union);  // find unique within collection
      }

      // run over all vertCoords and construct the union
      List<VertCoord> vertList = new ArrayList<>();
      VertCoord vc_union = null;
      for (GridDatasetInv.Grid grid : gridList) {
        VertCoord vc = grid.vc;
        if (vc == null) continue;
        if (vc_union == null)
          vc_union = new VertCoord(vc);
        else if (!vc_union.equalsData(vc)) {
//          System.out.printf("GridVariable %s has different vert coords in file %s %n", grid.getName(), grid.getFile());
          vertList.add(vc);
        }
      }
      if (vc_union != null) {
        VertCoord.normalize(vc_union, vertList); // add the other coords
        vertCoordUnion = VertCoord.findVertCoord(getVertCoords(), vc_union); // now find unique within collection
      }

      // run over all timeCoords and construct the union
      List<TimeCoord> timeList = new ArrayList<>();
      for (GridDatasetInv.Grid grid : gridList) {
        TimeCoord tc = grid.tc;
        timeList.add(tc);
      }
      // all time coordinates have the same run date
      TimeCoord tc_union = TimeCoord.makeUnion(timeList, getRunDate()); // add the other coords
      timeCoordUnion = TimeCoord.findTimeCoord(getTimeCoords(), tc_union); // now find unique within collection
    }

  }

  public Set<GridDatasetInv> getFiles() {
    HashSet<GridDatasetInv> fileSet = new HashSet<>();
    for (FmrInv.GridVariable grid :getGrids()) {
      for (GridDatasetInv.Grid inv : grid.getInventory())  {
        fileSet.add(inv.getFile());
      }
    }
    return fileSet;
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.server.cdmrfeature;

import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;
import ucar.unidata.geoloc.LatLonPoint;
import ucar.unidata.geoloc.LatLonPointImpl;
import ucar.unidata.geoloc.LatLonRect;

import java.util.Arrays;
import java.util.Formatter;

/**
 * Describe
 *
 * @author caron
 * @since 5/5/2015
 */
public class CdmrfBeanOld {
  public enum RequestType {
    capabilities, cdl, data, dataForm, form, header, ncml, stations
  }

  public enum ResponseType {
    csv, netcdf, ncstream, html, xml
  }

  public enum SpatialSelection {
    all, bb, point, stns
  }

  public enum TemporalSelection {
    all, range, point
  }

  // type of request
  private String req = "";

  // type of response
  private String accept = "";

  // comma delimited list of variable names
  private String variables; // (forms) all some
  private String var;

  //// spatial selection
  private String spatial; // (forms) all, bb, point, stns
  private TemporalSelection temporalSelection;

  // comma delimited list of station ids
  private String stn;

  // spatial extent - one or none
  private String bbox;
  private String west, east, south, north;
  private String latitude, longitude;

  //// time selection
  private String temporal; // (forms) all, range, point
  private SpatialSelection spatialSelection;

  // time range
  private String time_start, time_end, time_duration;
  private String time;

  // parsed quantities
  private DateRange dateRange;
  private DateType timePoint;
  private LatLonRect llbb;
  private LatLonPoint latlonPoint;
  private RequestType reqType = null;
  private ResponseType resType = null;

  private boolean fatal = false;
  private Formatter errs = new Formatter();

  boolean hasFatalError() {
    return fatal;
  }

  String getErrorMessage() {
    return errs.toString();
  }

  public LatLonRect getLatLonRect() {
    return (spatialSelection == SpatialSelection.bb) ? llbb : null;
  }

  DateRange getDateRange() {
    return dateRange;
  }

  public LatLonPoint getLatlonPoint() {
    return latlonPoint;
  }

  public DateType getTimePoint() {
    return timePoint;
  }

  public RequestType getRequestType() {
    if (reqType == null) {
      if (req.equalsIgnoreCase("capabilities")) reqType = RequestType.capabilities;
      else if (req.equalsIgnoreCase("cdl")) reqType = RequestType.cdl;
      else if (req.equalsIgnoreCase("data")) reqType = RequestType.data;
      else if (req.equalsIgnoreCase("dataForm")) reqType = RequestType.dataForm;
      else if (req.equalsIgnoreCase("form")) reqType = RequestType.form;
      else if (req.equalsIgnoreCase("header")) reqType = RequestType.header;
      else if (req.equalsIgnoreCase("ncml")) reqType = RequestType.ncml;
      else if (req.equalsIgnoreCase("stations")) reqType = RequestType.stations;
      else reqType = RequestType.data; // default
    }
    return reqType;
  }

  ResponseType getResponseType() {
    if (resType == null) {
      RequestType req = getRequestType();
      if (req == RequestType.capabilities) resType = ResponseType.xml;
      else if (req == RequestType.form) resType = ResponseType.html;
    }

    if (resType == null) {
      if (accept.equalsIgnoreCase("csv")) resType = ResponseType.csv;
      else if (accept.equalsIgnoreCase("ncstream")) resType = ResponseType.ncstream;
      else if (accept.equalsIgnoreCase("netcdf")) resType = ResponseType.netcdf;
      else if (accept.equalsIgnoreCase("xml")) resType = ResponseType.xml;
      else resType = ResponseType.ncstream; // default
    }

    return resType;
  }

  SpatialSelection getSpatialSelection() {
    return spatialSelection;
  }

  TemporalSelection getTemporalSelection() {
    return temporalSelection;
  }

  boolean validate() {
    RequestType reqType = getRequestType();
    if (reqType == null) {
      errs.format("form must req=[%s]%n", Arrays.toString(RequestType.values()));
      fatal = true;
      return !fatal;
    }

    switch (reqType) {
      case dataForm:
        parseVariablesForm();
        parseSpatialExtentForm();
        parseTemporalExtentForm();
        break;

      case data:
        parseVariablesForm();
        // fall through
      default:
        parseSpatialExtent();
        parseTimeExtent();
        if ((spatialSelection == null) && (stn != null))
          spatialSelection = SpatialSelection.stns;
    }

    return !fatal;
  }

  private void parseVariablesForm() {  // from the form
    if (variables == null) {
      errs.format("data request must have var=(all|some)%n");
      fatal = true;
      return;
    }

    if (variables.equalsIgnoreCase("all")) {
      setVar(null);
    }
  }

  private void parseSpatialExtentForm() { // from the form
    if (spatial == null) {
      errs.format("form must have spatial=(all|bb|point|stns)%n");
      fatal = true;
      return;
    }

    if (spatial.equalsIgnoreCase("all")) spatialSelection = SpatialSelection.all;
    else if (spatial.equalsIgnoreCase("bb")) spatialSelection = SpatialSelection.bb;
    else if (spatial.equalsIgnoreCase("point")) spatialSelection = SpatialSelection.point;
    else if (spatial.equalsIgnoreCase("stns")) spatialSelection = SpatialSelection.stns;

    if (spatialSelection == SpatialSelection.bb) {
      parseSpatialExtent();

    } else if (spatialSelection == SpatialSelection.point) {
      double lat = parseLat("latitude", latitude);
      double lon = parseLon("longitude", longitude);
      latlonPoint = new LatLonPointImpl(lat, lon);
    }

  }

  private void parseSpatialExtent() {
    if (bbox != null) {
      String[] s = bbox.split(",");
      if (s.length != 4) {
        errs.format("bbox must have form 'bbox=west,east,south,north'; found 'bbox=%s'%n", bbox);
        fatal = true;
        return;
      }
      west = s[0];
      east = s[1];
      south = s[2];
      north = s[3];
    }

    if ((west != null) || (east != null) || (south != null) || (north != null)) {
      if ((west == null) || (east == null) || (south == null) || (north == null)) {
        errs.format("All edges (west,east,south,north) must be specified; found west=%s east=%s south=%s north=%s %n", west, east, south, north);
        fatal = true;
        return;
      }
      double westd = parseLon("west", west);
      double eastd = parseLon("east", east);
      double southd = parseLat("south", south);
      double northd = parseLat("north", north);

      if (!fatal) {
        llbb = new LatLonRect(new LatLonPointImpl(southd, westd), new LatLonPointImpl(northd, eastd));
        spatialSelection = SpatialSelection.bb;
      }
    }
  }

  private double parseLat(String key, String value) {
    double lat = parseDouble(key, value);
    if (!Double.isNaN(lat)) {
      if ((lat > 90.0) || (lat < -90.0)) {
        errs.format("Illegal param= param='%s=%s' must be between +/- 90.0 %n", key, value);
        lat = Double.NaN;
        fatal = true;
      }
    }
    return lat;
  }

  private double parseLon(String key, String value) {
    return parseDouble(key, value);
  }

  private double parseDouble(String key, String value) {
    value = value.trim();
    try {
      return Double.parseDouble(value);
    } catch (NumberFormatException e) {
      errs.format("Illegal param='%s=%s' must be valid floating point number%n", key, value);
      fatal = true;
    }
    return Double.NaN;
  }

  ////////////////////////////////////////

  private void parseTemporalExtentForm() { // from the form
    if (temporal == null) {
      errs.format("form must have temporal=(all|range|point)%n");
      fatal = true;
      return;
    }

    if (temporal.equalsIgnoreCase("all")) temporalSelection = TemporalSelection.all;
    else if (temporal.equalsIgnoreCase("range")) temporalSelection = TemporalSelection.range;
    else if (temporal.equalsIgnoreCase("point")) temporalSelection = TemporalSelection.point;

    if (temporal.equalsIgnoreCase("range")) {
      try {
        parseTimeExtent();
      } catch (Throwable t) {
        errs.format("badly specified time range");
        fatal = true;
      }
    } else if (temporal.equalsIgnoreCase("point")) {
      timePoint = parseDate("time", time);
    }
  }

  private void parseTimeExtent() {
    DateType startDate = parseDate("time_start", time_start);
    DateType endDate = parseDate("time_end", time_end);
    TimeDuration duration = parseW3CDuration("time_duration", time_duration);

    // no range
    if ((startDate != null) && (endDate != null))
      dateRange = new DateRange(startDate, endDate, null, null);
    else if ((startDate != null) && (duration != null))
      dateRange = new DateRange(startDate, null, duration, null);
    else if ((endDate != null) && (duration != null))
      dateRange = new DateRange(null, endDate, duration, null);

    if (dateRange != null)
      temporalSelection = TemporalSelection.range;
  }

  public DateType parseDate(String key, String value) {
    if (value != null) {
      try {
        return new DateType(value, null, null);
      } catch (java.text.ParseException e) {
        errs.format("Illegal param='%s=%s'  must be valid ISO Date%n", key, value);
        fatal = true;
      }
    }
    return null;
  }

  public TimeDuration parseW3CDuration(String key, String value) {
    if (value != null) {
      try {
        return new TimeDuration(value);
      } catch (java.text.ParseException e) {
        errs.format("Illegal param='%s=%s'  must be valid ISO Duration%n", key, value);
        fatal = true;
      }
    }
    return null;
  }

  /////////////////////////////////////

  public void setAccept(String accept) {
    this.accept = accept;
  }

  public void setReq(String req) {
    this.req = req;
  }

  // variable names
  public void setVariables(String variables) {
    this.variables = variables;
  }

  public void setVar(String var) {
    this.var = var;
  }

  public String getVar() {
    return var;
  }

  public String[] getVarNames() {
    return (var == null) ? null : var.split(",");
  }

  //////// spatial

  public void setSpatial(String spatial) {
    this.spatial = spatial;
  }

  public void setStn(String stn) {
    this.stn = stn;
  }

  public String getStn() {
    return stn;
  }

  public String[] getStnNames() {
    if (!(spatialSelection == SpatialSelection.stns)) return null;
    return (stn == null) ? null : stn.split(",");
  }

  public void setBbox(String bbox) {
    this.bbox = bbox;
  }

  public void setWest(String west) {
    this.west = west;
  }

  public void setEast(String east) {
    this.east = east;
  }

  public void setSouth(String south) {
    this.south = south;
  }

  public void setNorth(String north) {
    this.north = north;
  }

  public void setLatitude(String latitude) {
    this.latitude = latitude;
  }

  public void setLongitude(String longitude) {
    this.longitude = longitude;
  }

  //////// temporal

  public void setTemporal(String temporal) {
    this.temporal = temporal;
  }

  public void setTime_start(String timeStart) {
    this.time_start = timeStart;
  }

  public void setTime_end(String timeEnd) {
    this.time_end = timeEnd;
  }

  public void setTime_duration(String timeDuration) {
    this.time_duration = timeDuration;
  }

  public void setTime(String time) {
    this.time = time;
  }

  @Override
  public String toString() {
    Formatter f = new Formatter();
    f.format("QueryBean: reqType=%s resType=%s", getRequestType(), getResponseType());

    if (spatialSelection == SpatialSelection.all)
      f.format(" spatialSelection=all;");
    else if (spatialSelection == SpatialSelection.bb)
      f.format(" bb=%s;", getLatLonRect());
    else if (spatialSelection == SpatialSelection.stns)
      f.format(" stns=%s;", getStn());

    if (temporalSelection == TemporalSelection.all)
      f.format(" temporalSelection=all;");
    else if (temporalSelection == TemporalSelection.range)
      f.format(" range=%s;", getDateRange());

    if (var != null)
      f.format(" vars=%s", var);

    return f.toString();
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.client.catalog;

import org.jdom2.Namespace;
import thredds.client.catalog.builder.AccessBuilder;
import thredds.client.catalog.builder.CatalogBuilder;
import thredds.client.catalog.builder.CatalogRefBuilder;
import thredds.client.catalog.builder.DatasetBuilder;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.util.URLnaming;

import javax.annotation.concurrent.Immutable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * A Client Catalog
 *
 * @author caron
 * @since 1/7/2015
 */
@Immutable
public class Catalog extends DatasetNode {
  static public final String CATALOG_NAMESPACE_10 = "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0";
  static public final Namespace defNS = Namespace.getNamespace(CATALOG_NAMESPACE_10);
  static public final String NJ22_NAMESPACE = "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";
  static public final String NJ22_NAMESPACE_HTTPS = "https://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";
  static public final Namespace ncmlNS = Namespace.getNamespace("ncml", NJ22_NAMESPACE);
  static public final Namespace ncmlNSHttps = Namespace.getNamespace("ncml", NJ22_NAMESPACE_HTTPS);
  static public final String XLINK_NAMESPACE = "http://www.w3.org/1999/xlink";
  static public final Namespace xlinkNS = Namespace.getNamespace("xlink", XLINK_NAMESPACE);
  static public final Namespace xsiNS = Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");

  // all of these are catalog only
  // public static final String CatalogScan = "CatalogScan";       // List<CatalogScan>
  public static final String DatasetHash = "DatasetHash";       // Map<String,Dataset>
  public static final String DatasetRoots = "DatasetRoots";     // List<DatasetRootConfig>
  public static final String Expires = "Expires";               // CalendarDate
  public static final String Services = "Services";             // List<Service>  (LOOK what about using Map ?)
  public static final String Version = "Version";               // String

  //////////////////////////////////////////////////////////////////////////////////////////
  private final URI baseURI;   // LOOK its possible we never want to use this. perhaps "location" instead ??

  public Catalog(URI baseURI, String name, Map<String, Object> flds, List<DatasetBuilder> datasets) {
    super(null, name, flds, datasets);
    this.baseURI = baseURI;

    Map<String, Dataset> datasetMap = new HashMap<>();
    addDatasetsToHash(getDatasetsLocal(), datasetMap);
    if (!datasetMap.isEmpty())
      flds.put(Catalog.DatasetHash, datasetMap);
  }

  private void addDatasetsToHash(List<Dataset> datasets, Map<String, Dataset> datasetMap) {
    if (datasets == null) return;
    for (Dataset ds : datasets) {
      String id = ds.getIdOrPath();
      if (id != null) datasetMap.put(id, ds);
      if (ds instanceof CatalogRef) continue; // dont recurse into catrefs
      addDatasetsToHash(ds.getDatasetsLocal(), datasetMap);
    }
  }

  public URI getBaseURI() {
    return baseURI;
  }

  public CalendarDate getExpires() {
    return (CalendarDate) flds.get(Catalog.Expires);
  }

  public String getVersion() {
    return (String) flds.get(Catalog.Version);
  }

  public List<Service> getServices() {
    List<Service> services = (List<Service>) flds.get(Catalog.Services);
    return services == null ? new ArrayList<>(0) : services;
  }

  public boolean hasService(String name) {
    for (Service s : getServices())
      if (s.getName().equalsIgnoreCase(name)) return true;
    return false;
  }

  public Service findService(String serviceName)  {
    if (serviceName == null) return null;
    List<Service> services = (List<Service>) flds.get(Catalog.Services);
    return findService(services, serviceName);
  }

  public Service findService(ServiceType type)  {
    if (type == null) return null;
    List<Service> services = (List<Service>) flds.get(Catalog.Services);
    return findService(services, type);
  }

  private Service findService(List<Service> services, String want)  {
    if (services == null) return null;
    for (Service s : services) {
      if (s.getName().equals(want)) return s;
    }
    for (Service s : services) {
      Service result = findService(s.getNestedServices(), want);
      if (result != null) return result;
    }
    return null;
  }

  private Service findService(List<Service> services, ServiceType type)  {
    if (services == null) return null;
    for (Service s : services) {
      if (s.getType() == type) return s;
    }
    for (Service s : services) {
      Service result = findService(s.getNestedServices(), type);
      if (result != null) return result;
    }
    return null;
  }

  public List<Property> getProperties() {
    List<Property> properties = (List<Property>) flds.get(Dataset.Properties);
    return properties == null ? new ArrayList<>(0) : properties;
  }

  public Dataset findDatasetByID( String id) {
    Map<String, Dataset> datasetMap = (Map<String, Dataset>) flds.get(Catalog.DatasetHash);
    return datasetMap == null ? null : datasetMap.get(id);
  }

  // get all datasets contained directly in this catalog
  public Iterable<Dataset> getAllDatasets() {
    List<Dataset> all = new ArrayList<>();
    addAll(this, all);
    return all;
  }

  private void addAll(DatasetNode node, List<Dataset> all) {
    all.addAll(node.getDatasetsLocal());
    for (DatasetNode nested : node.getDatasetsLocal())
      addAll(nested, all);
  }

  /**
   * Resolve reletive URIs, using the catalog's base URI. If the uriString is not reletive, then
   * no resolution is done. This also allows baseURI to be a file: scheme.
   *
   * @param uriString any url, reletive or absolute
   * @return resolved url string, or null on error
   * @throws java.net.URISyntaxException if uriString violates RFC 2396
   * @see java.net.URI#resolve
   */
  public URI resolveUri(String uriString) throws URISyntaxException {
    if (baseURI == null) return new URI(uriString);
    String resolved = URLnaming.resolve(baseURI.toString(), uriString);
    return new URI(resolved);
  }

  // look is this different than URLnaming ??
  public static URI resolveUri(URI baseURI, String uriString) throws URISyntaxException {
    URI want = new URI(uriString);
    if ((baseURI == null) || want.isAbsolute())
      return want;

    // gotta deal with file ourself
    String scheme = baseURI.getScheme();
    if ((scheme != null) && scheme.equals("file")) {
      String baseString = baseURI.toString();
      if ((uriString.length() > 0) && (uriString.charAt(0) == '#'))
        return new URI(baseString + uriString);
      int pos = baseString.lastIndexOf('/');
      if (pos > 0) {
        String r = baseString.substring(0, pos + 1) + uriString;
        return new URI(r);
      }
    }

    //otherwise let the URI class resolve it
    return baseURI.resolve(want);
  }

  public String getUriString() {
    URI baseURI = getBaseURI();
    return baseURI == null ? null : baseURI.toString();
  }

  //////////////////////////////////////////////////////////////////////////////////
  // from DeepCopyUtils, for subsetting

  public Catalog subsetCatalogOnDataset( Dataset dataset) {
    if ( dataset == null ) throw new IllegalArgumentException( "Dataset may not be null." );
    if ( dataset.getParentCatalog() != this ) throw new IllegalArgumentException( "Catalog must contain the dataset." );

    CatalogBuilder builder = new CatalogBuilder();

    URI docBaseUri = formDocBaseUriForSubsetCatalog( dataset );
    builder.setBaseURI(docBaseUri);
    builder.setName( dataset.getName());

    List<Service> neededServices = new ArrayList<>();
    DatasetBuilder topDs = copyDataset( null, dataset, neededServices, true );  // LOOK, cant set catalog as datasetNode parent

    for (Service s : neededServices)
      builder.addService(s);

    builder.addDataset( topDs );

    return builder.makeCatalog();
  }

  private DatasetBuilder copyDataset( DatasetBuilder parent, Dataset dataset, List<Service> neededServices, boolean copyInherited ) {

    neededServices.add(dataset.getServiceDefault());

    DatasetBuilder result;

    if ( dataset instanceof CatalogRef ) {
      CatalogRef catRef = (CatalogRef) dataset;
      CatalogRefBuilder catBuilder = new CatalogRefBuilder( parent);
      catBuilder.setHref( catRef.getXlinkHref());
      catBuilder.setTitle( catRef.getName());
      result = catBuilder;

    } else {
      result =  new DatasetBuilder(parent);

      List<Access> access = dataset.getLocalFieldAsList(Dataset.Access);  // dont expand
      for ( Access curAccess : access) {
        result.addAccess( copyAccess( result, curAccess, neededServices ));
      }

      List<Dataset> datasets = dataset.getLocalFieldAsList(Dataset.Datasets);  // dont expand
      for ( Dataset currDs : datasets) {
        result.addDataset( copyDataset( result, currDs, neededServices, copyInherited ));
      }
    }

    result.setName( dataset.getName() );
    result.transferMetadata(dataset, copyInherited);  // make a copy of all local metadata
    return result;
  }

  private AccessBuilder copyAccess( DatasetBuilder parent, Access access, List<Service> neededServices ) {
    neededServices.add(access.getService());  // LOOK may get dups
    return new AccessBuilder( parent, access.getUrlPath(), access.getService(), access.getDataFormatName(), access.getDataSize() );
  }

  private URI formDocBaseUriForSubsetCatalog( Dataset dataset ) {
    String catDocBaseUri = getUriString();
    String subsetDocBaseUriString = catDocBaseUri + "/" + ( dataset.getID() != null ? dataset.getID() : dataset.getName() );
    try {
      return new URI( subsetDocBaseUriString);
    } catch ( URISyntaxException e ) {   // This shouldn't happen. But just in case ...
      throw new IllegalStateException( "Bad document Base URI for new catalog [" + catDocBaseUri + "/" + (dataset.getID() != null ? dataset.getID() : dataset.getName()) + "].", e );
    }
  }

}


/* Copyright 2012, UCAR/Unidata.
   See the LICENSE file for more information.
*/

package dap4.servlet;

import dap4.core.data.DSP;
import dap4.core.util.DapContext;
import dap4.core.util.DapDump;
import dap4.core.util.DapException;
import dap4.core.util.DapUtil;
import dap4.dap4lib.ChunkInputStream;
import dap4.dap4lib.DapCodes;
import dap4.dap4lib.RequestMode;
import dap4.dap4lib.XURI;
import dap4.dap4lib.serial.D4DSP;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

/**
 * Provide a DSP interface to synthetic data (see Generator.java).
 */

public class SynDSP extends D4DSP
{
    static final protected boolean DEBUG = false;

    static protected final String[] SYNEXTENSIONS = new String[]{
            ".dmr", ".syn"
    };

    //////////////////////////////////////////////////
    // Instance variables

    protected byte[] raw = null; // Complete serialized binary databuffer

    //////////////////////////////////////////////////
    // Constructor(s)

    public SynDSP()
    {
        super();
    }

    //////////////////////////////////////////////////
    // DSP API

    /**
     * A path is a Synthetic path if it ends in .dmr or .syn
     *
     * @param path
     * @param context Any parameters that may help to decide.
     * @return true if this path appears to be processible by this DSP
     */
    public boolean dspMatch(String path, DapContext context)
    {
        for(String ext : SYNEXTENSIONS) {
            if(path.endsWith(ext))
                return true;
        }
        return false;
    }

    @Override
    public void close()
    {
    }

    @Override
    public SynDSP
    open(String filepath)
            throws DapException
    {
        // convert the relative path to real path
        assert this.context != null;
        if(filepath.startsWith("file:")) try {
            XURI xuri = new XURI(filepath);
            filepath = xuri.getPath();
        } catch (URISyntaxException use) {
            throw new DapException("Malformed filepath: " + filepath)
                    .setCode(DapCodes.SC_NOT_FOUND);
        }
        setLocation(filepath);
        // Read the .dmr/.syn file
        String document;
        try {
            try (FileInputStream stream = new FileInputStream(filepath);) {
                document = DapUtil.readtextfile(stream);
            }
        } catch (IOException ioe) {
            throw new DapException(ioe);
        }
        // Parse the dmr.
        this.dmr = parseDMR(document);

        // Use the Generator to generate synthetic data for this dmr.
        try {
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            ChunkWriter cw = new ChunkWriter(bos, RequestMode.DAP, ByteOrder.nativeOrder());
            Generator generator = new Generator(dmr, Value.ValueSource.RANDOM);
            generator.generate(null, cw, true, getChecksumMode());
            cw.close();
            bos.close();
            byte[] raw = bos.toByteArray();
            if(DEBUG)
                DapDump.dumpbytes(ByteBuffer.wrap(raw).order(getOrder()), true);
            ByteArrayInputStream bis = new ByteArrayInputStream(raw);
            ChunkInputStream crdr = new ChunkInputStream(bis, RequestMode.DAP, getOrder());
            // Skip the dmr
            crdr.readDMR();
            this.raw = DapUtil.readbinaryfile(crdr);
            super.build(dmr, this.raw, getOrder());
            return this;
        } catch (IOException ioe) {
            throw new DapException(ioe);
        }
    }

}


/*
 * Copyright (c) 2012-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package dap4.cdm.dsp;

import dap4.cdm.CDMTypeFcns;
import dap4.cdm.CDMUtil;
import dap4.cdm.NodeMap;
import dap4.core.data.DataCursor;
import dap4.core.dmr.*;
import dap4.core.util.Convert;
import dap4.core.util.DapContext;
import dap4.core.util.DapException;
import dap4.core.util.DapUtil;
import dap4.dap4lib.AbstractDSP;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.ma2.IndexIterator;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.dataset.*;
import ucar.nc2.util.CancelTask;

import java.io.IOException;
import java.util.*;

/**
 * Wrap CDM source (NetcdfDataset) to be a DSP
 * This basically means wrapping various (CDM)Array
 * object to look like DataVariable objects.
 * Currently only used on server side
 */

public class CDMDSP extends AbstractDSP
{

    //////////////////////////////////////////////////
    // Constants

    static protected final boolean DEBUG = false;
    static protected final boolean DUMPCDL = false;

    // NetcdfDataset enhancement to use: need only coord systems
    static protected Set<NetcdfDataset.Enhance> ENHANCEMENT = EnumSet.of(NetcdfDataset.Enhance.CoordSystems);

    static final protected String FILLVALUE = "_FillValue";

    //////////////////////////////////////////////////
    // Class variables

    static protected boolean nc4loaded = false;

    //////////////////////////////////////////////////
    // Class methods

    //////////////////////////////////////////////////
    // Instance variables

    protected NetcdfDataset ncdfile = null;
    protected boolean closed = false;
    protected DMRFactory dmrfactory = null;

    // Keep various context-dependent maps between
    // CDMNode instances and DapNode instances
    // package access

    // Variable map
    protected NodeMap<Variable, DapVariable> varmap = new NodeMap<>();

    // Variable <-> DapStructure (basetype) map; compound types only
    protected NodeMap<Variable, DapStructure> compoundmap = new NodeMap<>();

    // Map Variable <-> DapSequence vlen type
    protected NodeMap<Variable, DapSequence> vlenmap = new NodeMap<>();

    // Map All other kinds of CDMNode <-> DapNode
    protected NodeMap<CDMNode, DapNode> nodemap = new NodeMap<>();

    //////////////////////////////////////////////////
    // Constructor(s)

    public CDMDSP()
    {
    }

    public CDMDSP(String path)
            throws DapException
    {
        super();
        setLocation(path);
    }

    //////////////////////////////////////////////////
    // DSP Interface

    // This is intended to be the last DSP checked
    @Override
    public boolean dspMatch(String path, DapContext context)
    {
        return true;
    }

    /**
     * @param filepath - absolute path to a file
     * @return CDMDSP instance
     * @throws DapException
     */
    @Override
    public CDMDSP
    open(String filepath)
            throws DapException
    {
        try {
            NetcdfFile ncfile = createNetcdfFile(filepath, null);
            NetcdfDataset ncd = new NetcdfDataset(ncfile, ENHANCEMENT);
            return open(ncd);
        } catch (IOException ioe) {
            throw new DapException("CDMDSP: cannot process: " + filepath, ioe);
        }
    }

    /**
     * Provide an extra API for use in testing
     *
     * @param ncd
     * @return
     * @throws DapException
     */
    public CDMDSP open(NetcdfDataset ncd)
            throws DapException
    {
        assert this.context != null;
        this.dmrfactory = new DMRFactory();
        this.ncdfile = ncd;
        setLocation(this.ncdfile.getLocation());
        buildDMR();
        return this;
    }

    @Override
    public void close()
            throws IOException
    {
        if (this.ncdfile != null)
            this.ncdfile.close();
    }

    @Override
    public DataCursor
    getVariableData(DapVariable var)
            throws DapException
    {
        Variable cdmvar = this.varmap.get(var);
        if (cdmvar == null)
            throw new DapException("Unknown variable: " + var);
        CDMCursor vardata = (CDMCursor) super.getVariableData(var);
        if (vardata == null) {
            DataCursor.Scheme scheme = CDMCursor.schemeFor(var);
            try {
                vardata = new CDMCursor(scheme, this, var, null);
                vardata.setArray(cdmvar.read());
            } catch (IOException e) {
                throw new DapException(e);
            }
            super.addVariableData(var, vardata);
        }
        return vardata;
    }

    //////////////////////////////////////////////////
    // Abstract DSP Abstract methods

    //////////////////////////////////////////////////
    // CDMDSP Specific Accessors

    public NetcdfDataset getNetcdfDataset()
    {
        return this.ncdfile;
    }

    //////////////////////////////////////////////////
    // Nodemap Management

    /* Make sure that we use the proper
     * object in order to avoid identity
     * problems.
     */

    /**
     * Track generic CDMNode <-> DapNode
     */
    protected void
    recordNode(CDMNode cdm, DapNode dap)
    {
        assert this.nodemap.get(cdm) == null && this.nodemap.get(dap) == null;
        this.nodemap.put(cdm, dap);
    }

    /**
     * Track Variable <-> DapVariable
     */
    protected void
    recordVar(Variable cdm, DapVariable dap)
    {
        cdm = CDMUtil.unwrap(cdm);
        assert varmap.get(cdm) == null && varmap.get(dap) == null;
        varmap.put(cdm, dap);
    }

    /**
     * Track Variable <-> DapStructure
     */
    protected void
    recordStruct(Variable cdm, DapStructure dap)
    {
        cdm = CDMUtil.unwrap(cdm);
        assert this.nodemap.get(cdm) == null && this.nodemap.get(dap) == null;
        compoundmap.put(cdm, dap);
    }

    /**
     * Track Variable <-> DapSequence
     */
    protected void
    recordSeq(Variable cdm, DapSequence dap)
    {
        cdm = CDMUtil.unwrap(cdm);
        assert this.vlenmap.get(cdm) == null && this.vlenmap.get(dap) == null;
        vlenmap.put(cdm, dap);
    }

    /**
     * Lookup a cdmnode in the cdmnodemap.
     * Make sure that we use the proper
     * object in order to avoid identity
     * problems.
     *
     * @param cdmnode The CMDNode (variable, dimension, etc) to record
     * @return The DapNode to which the cdmnode is to be mapped or null.
     */
/*
    protected DapNode
    lookupNode(CDMNode cdmnode)
    {
        CDMSort sort = cdmnode.getSort();
        if(sort == CDMSort.VARIABLE || sort == CDMSort.STRUCTURE) {
            Variable basev = CDMUtil.unwrap((Variable) cdmnode);
            assert (basev != null) : "Unwrap() failed";
            cdmnode = (CDMNode) basev;
        }
        return this.nodemap.get(cdmnode);
    }
*/

    //////////////////////////////////////////////////

    /**
     * Extract the metadata from the NetcdfDataset
     * and build the DMR.
     */

    public void
    buildDMR()
            throws DapException
    {
        if (getDMR() != null)
            return;
        try {
            if (DUMPCDL) {
                System.out.println("writecdl:");
                this.ncdfile.writeCDL(System.out, false);
                System.out.flush();
            }
            // Use the file path to define the dataset name
            String name = this.ncdfile.getLocation();
            // Normalize the name
            name = DapUtil.canonicalpath(name);
            // Remove any path prefix
            int index = name.lastIndexOf('/');
            if (index >= 0)
                name = name.substring(index + 1, name.length());
            // Initialize the root dataset node
            setDMR((DapDataset) dmrfactory.newDataset(name).annotate(NetcdfDataset.class, this.ncdfile));
            // Map the CDM root group to this group
            recordNode(this.ncdfile.getRootGroup(), getDMR());
            getDMR().setBase(DapUtil.canonicalpath(this.ncdfile.getLocation()));

            // Now recursively build the tree. Start by
            // Filling the dataset with the contents of the ncfile
            // root group.
            fillgroup(getDMR(), this.ncdfile.getRootGroup());

            // Add an order index to the tree
            getDMR().sort();

            // Now locate the coordinate variables for maps

            /* Walk looking for VariableDS instances */
            processmappedvariables(this.ncdfile.getRootGroup());

            // Now set the view
            getDMR().finish();

        } catch (DapException e) {
            setDMR(null);
            throw new DapException(e);
        }
    }

    //////////////////////////////////////////////////
    // Actions

    protected void
    fillgroup(DapGroup dapgroup, Group cdmgroup)
            throws DapException
    {
        // Create decls in dap group for Dimensions
        for (Dimension cdmdim : cdmgroup.getDimensions()) {
            DapDimension dapdim = builddim(cdmdim);
        }
        // Create decls in dap group for Enumerations
        for (EnumTypedef cdmenum : cdmgroup.getEnumTypedefs()) {
            String name = cdmenum.getShortName();
            DapEnumeration dapenum = buildenum(cdmenum);
            dapenum.setShortName(name);
            dapgroup.addDecl(dapenum);
        }
        // Create decls in dap group for vlen induced Sequences
        // Do this before building compound types
        for (Variable cdmvar0 : cdmgroup.getVariables()) {
            Variable cdmvar = CDMUtil.unwrap(cdmvar0);
            buildseqtypes(cdmvar);
        }
        // Create decls in dap group for Compound Types
        for (Variable cdmvar0 : cdmgroup.getVariables()) {
            Variable cdmvar = CDMUtil.unwrap(cdmvar0);
            if (cdmvar.getDataType() != DataType.STRUCTURE
                    && cdmvar.getDataType() != DataType.SEQUENCE)
                continue;
            DapStructure struct = buildcompoundtype(cdmvar, dapgroup);
        }

        // Create decls in dap group for Variables
        for (Variable cdmvar0 : cdmgroup.getVariables()) {
            Variable cdmvar = CDMUtil.unwrap(cdmvar0);
            DapNode newvar = buildvariable(cdmvar, dapgroup, cdmvar.getDimensions());
        }
        // Create decls in dap group for subgroups
        for (Group subgroup : cdmgroup.getGroups()) {
            DapGroup newgroup = buildgroup(subgroup);
            dapgroup.addDecl(newgroup);
        }
        // Create decls in dap group for group-level attributes
        buildattributes(dapgroup, cdmgroup.getAttributes());
    }

    //////////////////////////////////////////////////
    // Declaration Builders

    protected DapDimension
    builddim(Dimension cdmdim)
            throws DapException
    {
        if (cdmdim.isVariableLength())
            throw new DapException("* dimensions not supported");
        DapDimension dapdim = null;
        long cdmsize = dapsize(cdmdim);
        String name = cdmdim.getShortName();
        if (name != null && name.length() == 0)
            name = null;
        boolean shared = cdmdim.isShared();
        if (!shared) {
            // Unlike the parser, since we are working
            // from a NetcdfDataset instance, there might
            // be multiple anonymous dimension objects
            // the same size. So, just go ahead and create
            // multiple instances.
            dapdim = (DapDimension) dmrfactory.newDimension(null, cdmsize);
            getDMR().addDecl(dapdim);
        } else { // Non anonymous; create in current group
            dapdim = (DapDimension) dmrfactory.newDimension(name, cdmsize);
            dapdim.setShared(true);
            if (cdmdim.isUnlimited()) {
                dapdim.setUnlimited(true);
            }
            Group cdmparent = cdmdim.getGroup();
            DapGroup dapparent = (DapGroup) this.nodemap.get(cdmparent);
            assert dapparent != null;
            assert (dapparent != null);
            dapparent.addDecl(dapdim);
        }
        recordNode(cdmdim, dapdim);
        return dapdim;
    }

    protected DapEnumeration
    buildenum(EnumTypedef cdmenum)
            throws DapException
    {
        // Set the enum's basetype
        DapType base = null;
        switch (cdmenum.getBaseType()) {
            case ENUM1:
                base = DapType.INT8;
                break;
            case ENUM2:
                base = DapType.INT16;
                break;
            case ENUM4:
            default:
                base = DapType.INT32;
                break;
        }
        DapEnumeration dapenum = (DapEnumeration) dmrfactory.newEnumeration(cdmenum.getShortName(), base);
        recordNode(cdmenum, dapenum);
        // Create the enum constants
        Map<Integer, String> ecvalues = cdmenum.getMap();
        for (Map.Entry<Integer, String> entry : ecvalues.entrySet()) {
            String name = entry.getValue();
            assert (name != null);
            int value = (int) entry.getKey();
            dapenum.addEnumConst(dmrfactory.newEnumConst(name, new Long(value)));
        }
        return dapenum;
    }

    protected DapStructure
    buildcompoundtype(Variable cdmvar, DapNode parent)
            throws DapException
    {
        cdmvar = CDMUtil.unwrap(cdmvar);
        DapStructure struct;
        if (cdmvar.getDataType() == DataType.STRUCTURE)
            struct = (DapStructure) dmrfactory.newStructure(cdmvar.getShortName());
        else if (cdmvar.getDataType() == DataType.SEQUENCE)
            struct = (DapStructure) dmrfactory.newSequence(cdmvar.getShortName());
        else
            throw new DapException("Internal error");
        struct.setParent(parent);
        recordStruct(cdmvar, struct);
        Structure cdmstruct = (Structure) cdmvar;
        List<Variable> fields = cdmstruct.getVariables();
        // recurse to create compound types of any fields c
        for (CDMNode node : fields) {
            Variable cdmfield = (Variable) node;
            List<Dimension> dimset = cdmfield.getDimensions();
            // Recurse on any nested compound types
            if (cdmfield.getDataType() == DataType.STRUCTURE
                    || cdmfield.getDataType() == DataType.SEQUENCE) {
                DapStructure fieldstruct = buildcompoundtype(cdmfield, struct);
            }
        }
        // Add the fields to this compound type
        for (CDMNode node : fields) {
            Variable cdmfield = (Variable) node;
            DapType basetype = null;
            switch (cdmfield.getDataType()) {
                default:
                    basetype = CDMTypeFcns.cdmtype2daptype(cdmfield.getDataType());
                    break;
                case STRUCTURE:
                case SEQUENCE:
                    basetype = compoundmap.get(cdmfield);// since no forward references
                    break;
            }
            // build the field variable
            List<Dimension> fielddims = cdmfield.getDimensions();
            DapVariable dapfield;
            if (CDMUtil.hasVLEN(cdmfield)) {
                // Get the fake sequence associated with this vlen
                DapSequence seq = vlenmap.get(cdmfield);
                List<Dimension> coredims = getCoreDimset(fielddims);
                // We need to build a variable whose basetype is the
                // fake sequence rather than the cdmfield type
                // First, build a variable using the existing cdmfield
                dapfield = buildvariable(cdmfield, struct, coredims);
                // Now modify it to use the fake sequence
                dapfield.setBaseType(seq);
            } else {
                dapfield = buildvariable(cdmfield, struct, fielddims);
            }
            struct.addField(dapfield);
        }
        return struct;
    }

    protected DapVariable
    buildvariable(Variable cdmbasevar, DapNode parent, List<Dimension> cdmdims)
            throws DapException
    {
        DapVariable dapvar = null;
        CDMSort sort = cdmbasevar.getSort();
        switch (sort) {
            case VARIABLE:
                switch (cdmbasevar.getDataType()) {
                    default:
                        dapvar = buildatomicvar(cdmbasevar, parent);
                        break;
                    case ENUM1:
                    case ENUM2:
                    case ENUM4:
                        dapvar = buildenumvar(cdmbasevar);
                        break;
                    case OPAQUE:
                        dapvar = buildopaquevar(cdmbasevar);
                        break;
                    case STRING:
                        dapvar = buildstringvar(cdmbasevar);
                        break;
                    case STRUCTURE:
                    case SEQUENCE:
                        assert false : "Internal error"; // How could this ever happen?
                        break;

                }
                builddimrefs(dapvar, cdmdims);
                break;
            case STRUCTURE:
                dapvar = buildstructvar(cdmbasevar);
                builddimrefs(dapvar, cdmdims);
                break;
            case SEQUENCE:
            default:
                assert false : "Internal Error";
        }
        if (parent != null)
            addToParent(parent, dapvar);
        return dapvar;
    }

    protected DapVariable
    buildatomicvar(Variable cdmvar, DapNode parent)
            throws DapException
    {
        // Atomic => not opaque and not enum
        DapType basetype = CDMTypeFcns.cdmtype2daptype(cdmvar.getDataType());
        if (basetype == null)
            throw new DapException("DapFile: illegal CDM variable base type: " + cdmvar.getDataType());
        DapVariable dapvar = (DapVariable) dmrfactory.newVariable(cdmvar.getShortName(), basetype);
        recordVar(cdmvar, dapvar);
        buildattributes(dapvar, cdmvar.getAttributes());
        if (CDMUtil.hasVLEN(cdmvar)) {
            // Get the fake sequence associated with this vlen
            DapSequence seq = vlenmap.get(cdmvar);
            List<Dimension> coredims = getCoreDimset(cdmvar.getDimensions());
            // We need to build a variable whose basetype is the
            // fake sequence rather than the cdmfield type
            // First, build a variable using the existing cdmfield
            // Now modify it to use the fake sequence
            dapvar.setBaseType(seq);
        }
        return dapvar;
    }

    protected DapVariable
    buildopaquevar(Variable cdmvar)
            throws DapException
    {
        assert (cdmvar.getDataType() == DataType.OPAQUE) : "Internal error";
        DapVariable dapvar = (DapVariable) dmrfactory.newVariable(cdmvar.getShortName(), DapType.OPAQUE);
        recordVar(cdmvar, dapvar);
        buildattributes(dapvar, cdmvar.getAttributes());
        Object osize = cdmvar.annotation(UCARTAGOPAQUE);
        if (osize != null) {
            dapvar.addXMLAttribute(UCARTAGOPAQUE, osize.toString());
        }
        return dapvar;
    }

    protected DapVariable
    buildstringvar(Variable cdmvar)
            throws DapException
    {
        assert (cdmvar.getDataType() == DataType.STRING) : "Internal error";
        DapVariable dapvar = (DapVariable) dmrfactory.newVariable(cdmvar.getShortName(), DapType.STRING);
        recordVar(cdmvar, dapvar);
        buildattributes(dapvar, cdmvar.getAttributes());
        return dapvar;
    }

    protected DapVariable
    buildenumvar(Variable cdmvar)
            throws DapException
    {
        assert (
                cdmvar.getDataType() == DataType.ENUM1
                        || cdmvar.getDataType() == DataType.ENUM2
                        || cdmvar.getDataType() == DataType.ENUM4
        ) : "Internal error";

        // Now, we need to locate the actual enumeration decl
        EnumTypedef enumdef = cdmvar.getEnumTypedef();
        EnumTypedef trueenumdef = findMatchingEnum(enumdef);
        // Modify the cdmvar
        cdmvar.setEnumTypedef(trueenumdef);
        // Now, map to a DapEnumeration
        DapEnumeration dapenum = (DapEnumeration) this.nodemap.get(trueenumdef);
        assert (dapenum != null);
        DapVariable dapvar = (DapVariable) dmrfactory.newVariable(cdmvar.getShortName(), dapenum);
        recordVar(cdmvar, dapvar);
        buildattributes(dapvar, cdmvar.getAttributes());
        return dapvar;
    }

    protected DapVariable
    buildstructvar(Variable cdmvar)
            throws DapException
    {
        assert (cdmvar.getDataType() == DataType.STRUCTURE) : "Internal error";
        // Find the DapStructure that is the basetype for this var
        DapStructure struct = compoundmap.get(cdmvar);
        assert struct != null : "Internal Error";
        DapVariable dapvar = (DapVariable) dmrfactory.newVariable(cdmvar.getShortName(), struct);
        recordVar(cdmvar, dapvar);
        buildattributes(dapvar, cdmvar.getAttributes());
        return dapvar;
    }

    /* Create a sequence from a variable with a
       variable length last dimension.
       Suppose we have cdm equivalent to this:
            T var[d1]...[dn]][*]
       We convert to the following
           <Sequence name="var">
             <T name="var"/>
             <Dim name=d1/>
             ...
             <Dim name=dn/>
           </Sequence>
    */

    protected DapSequence
    buildseqtype(Variable cdmvar)
            throws DapException
    {
        cdmvar = CDMUtil.unwrap(cdmvar);
        assert (CDMUtil.hasVLEN(cdmvar));
        DataType dt = cdmvar.getDataType();
        DapType daptype = CDMTypeFcns.cdmtype2daptype(dt);
        DapSequence seq = (DapSequence) dmrfactory.newSequence(cdmvar.getShortName());
        // fill DapSequence with a single field; note that the dimensions
        // are elided because they will attach to the sequence variable,
        // not the field
        DapVariable field = dmrfactory.newVariable(cdmvar.getShortName(), daptype);
        seq.addField(field);
        field.setParent(seq);
        recordSeq(cdmvar, seq);
        return seq;
    }

    /**
     * Walk this variable, including fields, to construct sequence types
     * for any contained vlen dimensions
     *
     * @param cdmvar variable to walk
     */

    protected void
    buildseqtypes(Variable cdmvar)
            throws DapException
    {
        if (CDMUtil.hasVLEN(cdmvar)) {
            buildseqtype(cdmvar);
        }
        if (cdmvar.getDataType() == DataType.STRUCTURE
                || cdmvar.getDataType() == DataType.SEQUENCE) {
            Structure struct = (Structure) cdmvar;
            List<Variable> fields = struct.getVariables();
            for (int i = 0; i < fields.size(); i++) {
                Variable field = fields.get(i);
                buildseqtypes(field); // recurse for inner vlen dims
            }
        }
    }

    protected void
    buildattributes(DapNode node, List<Attribute> attributes)
            throws DapException
    {
        for (Attribute attr : attributes) {
            if (!suppress(attr.getShortName())) {
                DapAttribute dapattr = buildattribute(attr);
                node.addAttribute(dapattr);
            }
        }
    }

    protected DapAttribute
    buildattribute(Attribute cdmattr)
            throws DapException
    {
        DapType attrtype = CDMTypeFcns.cdmtype2daptype(cdmattr.getDataType());
        EnumTypedef cdmenum = cdmattr.getEnumType();
        boolean enumfillvalue = (cdmattr.getShortName().equals(FILLVALUE)
                && cdmenum != null);
        DapEnumeration dapenum = null;

        // We need to handle _FillValue specially if the
        // the variable is enum typed.
        if (enumfillvalue) {
            cdmenum = findMatchingEnum(cdmenum);
            // Make sure the cdm attribute has type enumx
            if (!cdmenum.getBaseType().isEnum())
                throw new DapException("CDM _FillValue attribute type is not enumX");
            // Modify the attr
            cdmattr.setEnumType(cdmenum);
            // Now, map to a DapEnumeration
            dapenum = (DapEnumeration) this.nodemap.get(cdmenum);
            if (dapenum == null)
                throw new DapException("Illegal CDM variable attribute type: " + cdmenum);
            attrtype = dapenum;
        }
        if (attrtype == null)
            throw new DapException("DapFile: illegal CDM variable attribute type: " + cdmattr.getDataType());
        DapAttribute dapattr = (DapAttribute) dmrfactory.newAttribute(cdmattr.getShortName(), attrtype);
        recordNode(cdmattr, dapattr);
        // Transfer the values
        Array values = cdmattr.getValues();
        if (!validatecdmtype(cdmattr.getDataType(), values.getElementType()))
            throw new DapException("Attr type versus attribute data mismatch: " + values.getElementType());
        IndexIterator iter = values.getIndexIterator();
        String[] valuelist = null;
        Object vec = CDMTypeFcns.createVector(cdmattr.getDataType(), values.getSize());
        for (int i = 0; iter.hasNext(); i++) {
            java.lang.reflect.Array.set(vec, i, iter.next());
        }
        valuelist = (String[]) Convert.convert(DapType.STRING, attrtype, vec);
        dapattr.setValues(valuelist);
        return dapattr;
    }

    /**
     * Assign dimensions to a variable
     *
     * @param dapvar  The variable to which we wish to assign dimensions
     * @param cdmdims The cdm dimensions from which we will find the dimension info
     */
    protected void
    builddimrefs(DapVariable dapvar, List<Dimension> cdmdims)
            throws DapException
    {
        if (cdmdims == null || cdmdims.size() == 0)
            return;
        // It is unfortunately the case that the dimensions
        // associated with the variable are not
        // necessarily the same object as those dimensions
        // as declared, so we need to use a non-trivial
        // matching algorithm.
        for (Dimension cdmdim : cdmdims) {
            DapDimension dapdim = null;
            if (cdmdim.isShared()) {
                Dimension declareddim = finddimdecl(cdmdim);
                if (declareddim == null)
                    throw new DapException("Unprocessed cdm dimension: " + cdmdim);
                dapdim = (DapDimension) this.nodemap.get(declareddim);
                assert dapdim != null;
            } else if (cdmdim.isVariableLength()) {// ignore
                continue;
            } else {//anonymous
                dapdim = builddim(cdmdim);
            }
            assert (dapdim != null) : "Internal error";
            dapvar.addDimension(dapdim);
        }
    }

    protected void
    processmappedvariables(Group g)
            throws DapException
    {
        for (Variable v0 : g.getVariables()) {
            Variable cdmvar = CDMUtil.unwrap(v0);
            if (cdmvar == null)
                throw new DapException("NetcdfDataset synthetic variable: " + v0);
            DapNode dapvar = this.varmap.get(cdmvar);
            if (dapvar == null)
                throw new DapException("Unknown variable: " + cdmvar);
            if (!(dapvar instanceof DapVariable))
                throw new DapException("CDMVariable not mapping to dap variable: " + cdmvar);
            buildmaps((DapVariable) dapvar, v0);
        }
    }

    /**
     * @param dapvar The variable to which we wish to assign maps
     * @param var    The NetcdfDataset variable from which to extract coord system
     */
    protected void
    buildmaps(DapVariable dapvar, Variable var)
            throws DapException
    {
        // See if this cdm variable has one (or more) coordinate system
        List<CoordinateSystem> css = null;
        if (var.getSort() == CDMSort.VARIABLE) {
            VariableDS vds = (VariableDS) var;
            css = vds.getCoordinateSystems();
        } else {
            StructureDS sds = (StructureDS) var;
            css = sds.getCoordinateSystems();
        }
        if (css != null && css.size() > 0) {
            // Not sure what to do with multiple coordinate systems
            // For now, only use the first
            CoordinateSystem coordsystems = css.get(0);
            for (CoordinateAxis axis : coordsystems.getCoordinateAxes()) {
                // First step is to find the dap variable
                // corresponding to the map
                VariableDS vds = (VariableDS) axis.getOriginalVariable();
                if (vds != null) {
                    Variable v = CDMUtil.unwrap(vds);
                    if (v != null) {
                        DapVariable mapvar = varmap.get(v);
                        if (mapvar == null)
                            throw new DapException("Illegal map variable:" + v.toString());
                        if (!mapvar.isAtomic())
                            throw new DapException("Non-atomic map variable:" + v.toString());
                        // Ignore maps where the map variable is inside this scope
                        /*
                            if(!mapvar.isTopLevel()) {
                            DapNode parent = mapvar.getContainer();
                            switch (parent.getSort()) {
                            case SEQUENCE:
                            case STRUCTURE:
                                if(dapvar.getBaseType() == parent) // Do we need to do transitive closure?
                                    throw new DapException("Map var cannot be in same structure as map");
                                break;
                            default:
                                assert false : "Unexpected container type";
                            }
                            */
                        DapMap map = (DapMap) dmrfactory.newMap(mapvar);
                        dapvar.addMap(map);
                    }
                }
            }
        }
    }

    protected DapGroup
    buildgroup(Group cdmgroup)
            throws DapException
    {
        DapGroup dapgroup = (DapGroup) dmrfactory.newGroup(cdmgroup.getShortName());
        recordNode(cdmgroup, dapgroup);
        dapgroup.setShortName(cdmgroup.getShortName());
        fillgroup(dapgroup, cdmgroup);
        return dapgroup;
    }

    //////////////////////////////////////////////////
    // Utilities

    /**
     * Unfortunately, the CDM Iosp does not
     * actually use the declared enums. Rather,
     * for every enum type'd variable, a new
     * enum decl is defined. So, we need
     * to find the original enum decl that matches
     * the variable's enum.
     */

    protected EnumTypedef
    findMatchingEnum(EnumTypedef varenum)
            throws DapException
    {
        List<EnumTypedef> candidates = new ArrayList<>();
        for (Map.Entry<DapNode, CDMNode> entry : this.nodemap.getCDMMap().entrySet()) {
            CDMNode cdmnode = entry.getValue();
            if (cdmnode.getSort() != CDMSort.ENUMERATION)
                continue;
            // Compare the enumeration (note names will differ)
            EnumTypedef target = (EnumTypedef) cdmnode;
            /* Ideally, we should test the types of the enums,
               but, unfortunately, the var enum is always enum4.
            if(target.getBaseType() != varenum.getBaseType())
                continue;
            */
            Map<Integer, String> targetmap = target.getMap();
            Map<Integer, String> varmap = varenum.getMap();
            if (targetmap.size() != varmap.size())
                continue;
            boolean match = true; // until otherwise shown
            for (Map.Entry<Integer, String> tpair : targetmap.entrySet()) {
                String tname = tpair.getValue();
                int value = (int) tpair.getKey();
                boolean found = false;
                for (Map.Entry<Integer, String> vpair : varmap.entrySet()) {
                    if (tname.equals(vpair.getValue()) && value == (int) vpair.getKey()) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    match = false;
                    break;
                }
            }
            if (!match)
                continue;

            // Save it unless it is shadowed by a closer enum
            boolean shadowed = false;
            for (EnumTypedef etd : candidates) {
                if (shadows(etd.getGroup(), target.getGroup())) {
                    shadowed = true;
                    break;
                }
            }
            if (!shadowed)
                candidates.add(target);
        }

        switch (candidates.size()) {
            case 0:
                throw new DapException("CDMDSP: No matching enum type decl: " + varenum.getShortName());
            case 1:
                break;
            default:
                throw new DapException("CDMDSP: Multiple matching enum type decls: " + varenum.getShortName());
        }
        return candidates.get(0);
    }

    protected boolean
    shadows(Group parent, Group child)
    {
        if (child == parent)
            return true;
        Group candidate = child;
        do {
            candidate = candidate.getGroup();
        } while (candidate != null && candidate != parent);
        return (candidate == parent);
    }

    // Convert cdm size to DapDimension size
    protected long
    dapsize(Dimension cdmdim)
    {
        assert (!cdmdim.isVariableLength());
        return (long) cdmdim.getLength();
    }

    protected boolean
    validatecdmtype(DataType datatype, Class typeclass)
    {
        switch (datatype) {
            case CHAR:
                return typeclass == char.class;
            case BYTE:
            case UBYTE:
                return typeclass == byte.class;
            case SHORT:
            case USHORT:
                return typeclass == short.class;
            case INT:
            case UINT:
                return typeclass == int.class;
            case LONG:
            case ULONG:
                return typeclass == long.class;
            case FLOAT:
                return typeclass == float.class;
            case DOUBLE:
                return typeclass == double.class;
            case STRING:
                return typeclass == String.class;
            case OPAQUE:
                return typeclass == Byte[].class;
            // For these, return the integer basetype
            case ENUM1:
                return typeclass == byte.class;
            case ENUM2:
                return typeclass == short.class;
            case ENUM4:
                return typeclass == int.class;

            // Undefined
            case SEQUENCE:
            case STRUCTURE:
            default:
                break;
        }
        return false;
    }

    protected Dimension
    finddimdecl(Dimension dimref)
    {
        // Search on the full name, but be careful,
        // the rule is that the declared dimension's fqn
        // must be a prefix of the dimension reference.
        for (Map.Entry<DapNode, CDMNode> entry : this.nodemap.getCDMMap().entrySet()) {
            if (entry.getValue().getSort() != CDMSort.DIMENSION)
                continue;
            Dimension d = (Dimension) entry.getValue();
            if (isDimDeclFor(d, dimref))
                return d;
        }
        return null;
    }

    protected boolean isDimDeclFor(Dimension decl, Dimension ref)
    {
        if(!decl.isShared())
            return false; // Has no name
        // First check shortname and size
        if (!decl.getShortName().equals(ref.getShortName()))
            return false;
        if (decl.getLength() != ref.getLength())
            return false;
        // Make sure they are in the same group
        String dprefix = decl.getGroup().getFullName();
        String rprefix = ref.getGroup().getFullName();
        return (dprefix.equals(rprefix));
    }

    /*protected String
    fixvalue(Object o, DapType datatype)
    {
        TypeSort atype = datatype.getTypeSort();
        if(o instanceof Character) {
            long l = (long) ((Character) o).charValue();
            if(atype.isUnsigned())
                l = l & 0xffL;
            o = Long.valueOf(l);
        } else if(o instanceof Float || o instanceof Double) {
            if(atype == TypeSort.Float32)
                o = Float.valueOf(((Number) o).floatValue());
            else if(atype == TypeSort.Float64)
                o = Double.valueOf(((Number) o).doubleValue());
            else
                assert false : "Internal error";
        } else if(o instanceof Number) {
            long l = ((Number) o).longValue();
            switch (atype) {
            case Char:
            case UInt8:
                l = l & 0xffL;
                break;
            case UInt16:
                l = l & 0xffffL;
                break;
            case UInt32:
                l = l & 0xffffffffL;
                break;
            default:
                break;
            }
            o = Long.valueOf(l);
        } else if(o instanceof String) {
            o = o.toString();
        } else if(o instanceof ByteBuffer) {
            // leave it unchanged
        } else if(o instanceof byte[]) {
            o = ByteBuffer.wrap((byte[]) o);
        } else if(o instanceof Byte[]) {
            Byte[] ob = (Byte[]) o;
            byte[] bb = new byte[ob.length];
            for(int i = 0; i < bb.length; i++) {
                bb[i] = (byte) ob[i];
            }
            o = ByteBuffer.wrap(bb);
        } //else { // leave it unchanged
        return o;
    } */

    //////////////////////////////////////////////////

    protected NetcdfFile
    createNetcdfFile(String location, CancelTask canceltask)
            throws DapException
    {
        try {
            NetcdfFile ncfile = NetcdfFile.open(location, -1, canceltask, getContext());
            return ncfile;
        } catch (DapException de) {
            if (DEBUG)
                de.printStackTrace();
            throw de;
        } catch (Exception e) {
            if (DEBUG)
                e.printStackTrace();
            throw new DapException(e);
        }
    }

    //////////////////////////////////////////////////
    // Utilities

    /**
     * Strip vlen dimensions from a set of dimensions
     *
     * @param dimset
     * @return subset of dimset with (trailing) vlen removed
     * @throws DapException
     */
    static List<Dimension>
    getCoreDimset(List<Dimension> dimset)
            throws DapException
    {
        if (dimset == null) return null;
        List<Dimension> core = new ArrayList<>();
        int pos = -1;
        int count = 0;
        for (int i = 0; i < dimset.size(); i++) {
            if (dimset.get(i).isVariableLength()) {
                pos = i;
                count++;
            } else
                core.add(dimset.get(i));
        }
        if ((pos != dimset.size() - 1) || count > 1)
            throw new DapException("Unsupported use of (*) Dimension");
        return core;
    }

    /**
     * Some attributes that are added by the NetcdfDataset
     * need to be kept out of the DMR. This function
     * defines that set.
     *
     * @param attrname A non-escaped attribute name to be tested for suppression
     * @return true if the attribute should be suppressed, false otherwise.
     */
    protected boolean suppress(String attrname)
    {
        if (attrname.startsWith("_Coord")) return true;
        if (attrname.equals(CDM.UNSIGNED))
            return true;
        return false;
    }

    protected void
    addToParent(DapNode parent, DapVariable dapvar)
            throws DapException
    {
        assert (parent != null);
        switch (parent.getSort()) {
            case GROUP:
            case DATASET:
                ((DapGroup) parent).addDecl(dapvar);
                break;
            case SEQUENCE:
            case STRUCTURE:
                dapvar.setParent(parent);
                break;
            default:
                assert (false) : "Internal error";
        }
    }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ui.grib;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.featurecollection.FeatureCollectionConfig;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.grib.collection.GribCdmIndex;
import ucar.nc2.grib.collection.GribCollectionImmutable;
import ucar.nc2.ui.widget.BAMutil;
import ucar.nc2.ui.widget.IndependentWindow;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.ui.widget.TextHistoryPane;
import ucar.unidata.util.StringUtil2;
import ucar.util.prefs.PreferencesExt;
import ucar.util.prefs.ui.BeanTable;

import javax.swing.*;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
import java.util.List;

/**
 * Rewrite GRIB files - track stats etc
 *
 * @author caron
 * @since 8/11/2014
 */
public class GribRewritePanel extends JPanel {
  static private final Logger logger = LoggerFactory.getLogger(GribRewritePanel.class);

  private PreferencesExt prefs;

  private BeanTable ftTable;
  private JSplitPane split;
  private TextHistoryPane dumpTA;
  private IndependentWindow infoWindow;

  public GribRewritePanel(PreferencesExt prefs, JPanel buttPanel) {
    this.prefs = prefs;

    ftTable = new BeanTable(FileBean.class, (PreferencesExt) prefs.node("FeatureDatasetBeans"), false);
    /* ftTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        setSelectedFeatureDataset(ftb);
      }
    }); */

    AbstractAction calcAction = new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        calcAverage();
      }
    };
    BAMutil.setActionProperties(calcAction, "Dataset", "calc storage", false, 'C', -1);
    BAMutil.addActionToContainer(buttPanel, calcAction);

    PopupMenu varPopup = new PopupMenu(ftTable.getJTable(), "Options");
    varPopup.addAction("Open as NetcdfFile", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        if (ftb == null) return;
        GribRewritePanel.this.firePropertyChange("openNetcdfFile", null, ftb.getPath());
      }
    });

    varPopup.addAction("Open as GridDataset", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        if (ftb == null) return;
        GribRewritePanel.this.firePropertyChange("openGridDataset", null, ftb.getPath());
      }
    });

    varPopup.addAction("Open in Grib2Data", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        if (ftb == null) return;
        GribRewritePanel.this.firePropertyChange("openGrib2Data", null, ftb.getPath());
      }
    });

    varPopup.addAction("Open in Grib1Data", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        if (ftb == null) return;
        GribRewritePanel.this.firePropertyChange("openGrib1Data", null, ftb.getPath());
      }
    });

    /* varPopup.addAction("Show Report on selected rows", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        List<FileBean> selected = ftTable.getSelectedBeans();
        Formatter f = new Formatter();
        for (FileBean bean : selected) {
          bean.toString(f, false);
        }
        dumpTA.setText(f.toString());
      }
    });

    varPopup.addAction("Run Coverage Classifier", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        FileBean ftb = (FileBean) ftTable.getSelectedBean();
        if (ftb == null) return;
        dumpTA.setText(ftb.runClassifier());
      }
    });  */

    // the info window
    TextHistoryPane infoTA = new TextHistoryPane();
    infoWindow = new IndependentWindow("Extra Information", BAMutil.getImage("netcdfUI"), infoTA);
    infoWindow.setBounds((Rectangle) prefs.getBean("InfoWindowBounds", new Rectangle(300, 300, 500, 300)));

    dumpTA = new TextHistoryPane();
    split = new JSplitPane(JSplitPane.VERTICAL_SPLIT, false, ftTable, dumpTA);
    split.setDividerLocation(prefs.getInt("splitPos", 500));

    setLayout(new BorderLayout());
    add(split, BorderLayout.CENTER);
  }

  public PreferencesExt getPrefs() {
    return prefs;
  }

  public void save() {
    ftTable.saveState(false);
    prefs.putInt("splitPos", split.getDividerLocation());
    prefs.putBeanObject("InfoWindowBounds", infoWindow.getBounds());
  }

  public void clear() {
    ftTable.setBeans(new ArrayList()); // clear
  }

  public boolean setScanDirectory(String dirName) {
    clear();

    //repaint();
    Formatter errlog = new Formatter();
    List<FileBean> beans = scan(dirName, errlog);
    if (beans.size() == 0)  {
      dumpTA.setText(errlog.toString());
      return false;
    }

    ftTable.setBeans(beans);
    //repaint();
    return true;
  }

  private void setSelectedFeatureDataset(FileBean ftb) {
    dumpTA.setText(ftb.toString());
    dumpTA.gotoTop();
  }

  private void calcAverage() {
    Formatter f = new Formatter();
    calcAverage(null, f);
    calcAverage("grib1", f);
    calcAverage("grib2", f);

    dumpTA.setText(f.toString());
    dumpTA.gotoTop();
  }

  private void calcAverage(String what, Formatter f) {

    double totalRecords = 0.0;
    double ratio = 0.0;
    List<FileBean> beans = ftTable.getBeans();
    for (FileBean bean : beans) {
      if (bean.getNdups() > 0) continue;
      if (bean.getRatio() == 0) continue;
      if (what != null && (!bean.getPath().contains(what))) continue;
      totalRecords += bean.getNrecords();
      ratio += bean.getNrecords() * bean.getRatio();
    }

    double weightedAvg =  ratio / totalRecords;
    if (what != null) f.format("%n%s%n", what);
    f.format("Weighted average ratio = %f%n", weightedAvg);
    f.format("Total # grib records = %f%n", totalRecords);
  }

  ///////////////

  public java.util.List<FileBean> scan(String top, Formatter errlog) {

    List<FileBean> result = new ArrayList<>();

    File topFile = new File(top);
    if (!topFile.exists()) {
      errlog.format("File %s does not exist", top);
      return result;
    }

    if (topFile.isDirectory())
      scanDirectory(topFile, false, result, errlog);
    else {
      FileBean fdb = null;
      try {
        fdb = new FileBean(topFile);
      } catch (IOException e) {
        System.out.printf("FAIL, skip %s%n", topFile.getPath());
      }
      result.add(fdb);
    }

    return result;
  }

  public static class FileFilterFromSuffixes implements FileFilter {
    String[] suffixes;
    public FileFilterFromSuffixes(String suffixes) {
      this.suffixes = suffixes.split(" ");
    }

    @Override
    public boolean accept(File file) {
      for (String s: suffixes)
        if (file.getPath().endsWith(s)) return true;
      return false;
    }
  }

  private void scanDirectory(File dir, boolean subdirs, java.util.List<FileBean> result, Formatter errlog) {
    if ((dir.getName().equals("exclude")) || (dir.getName().equals("problem")))return;

    File[] gribFilesInDir = dir.listFiles(new FileFilterFromSuffixes("grib1 grib2"));
    if (gribFilesInDir == null) {
      // File.listFiles() returns null instead of throwing an exception. Dumb.
      throw new RuntimeException(String.format("Either an I/O error occurred, or \"%s\" is not a directory.", dir));
    }

    List<File> files = new ArrayList<>();
    for (File gribFile : gribFilesInDir) {
      if (!gribFile.isDirectory()) {
        files.add(gribFile);
      }
    }

    // eliminate redundant files
    // ".Z", ".zip", ".gzip", ".gz", or ".bz2"
    if (files.size() > 0) {
      Collections.sort(files);
      List<File> files2 = new ArrayList<>(files);

      File prev = null;
      for (File f : files) {
        String name = f.getName();
        String stem = stem(name);
        if (prev != null) {
          if (name.endsWith(".ncml")) {
            if (prev.getName().equals(stem) || prev.getName().equals(stem + ".grib2"))
               files2.remove(prev);
          }
        }
        prev = f;
      }

      // do the remaining
      for (File f : files2) {
        try {
          result.add(new FileBean(f));
        } catch (IOException e) {
          System.out.printf("FAIL, skip %s%n", f.getPath());
        }
      }
    }

    // do subdirs
    if (subdirs) {
      for (File f : dir.listFiles()) {
        if (f.isDirectory() && !f.getName().equals("exclude"))
          scanDirectory(f, subdirs, result, errlog);
      }
    }

  }

  private String stem(String name) {
    int pos = name.lastIndexOf('.');
    return (pos > 0) ? name.substring(0, pos) : name;
  }

  private static final boolean debug = true;

  public class FileBean {
    private File f;
    String fileType;
    long cdmData2D, nc4Data2D, nc4Size;
    int nrecords, nvars, ndups;

    // no-arg constructor
    public FileBean() {
    }

    public FileBean(File f) throws IOException {
      this.f = f;

      if (debug) System.out.printf(" fileScan=%s%n", getPath());
      try (NetcdfDataset ncd = NetcdfDataset.openDataset(getPath())) {
        fileType = ncd.getFileTypeId();
        cdmData2D = countCdmData2D(ncd);
        countGribData2D(f);
        countNc4Data2D(f);
      }
    }

    public String getPath() {
      String p = f.getPath();
      return StringUtil2.replace(p, "\\", "/");
    }

    public String getFileType() {
      return fileType;
    }

    public double getGribSizeM() {
      return ((double) f.length()) / 1000 /1000;
      /* Formatter fm = new Formatter();
      //long size = f.length();
      //if (size > 10 * 1000 * 1000) fm.format("%6.1f M", ((float) size) / 1000 / 1000);
      //else if (size > 10 * 1000) fm.format("%6.1f K", ((float) size) / 1000);
      //else fm.format("%d", size);
      fm.format("%,-15d", f.length() / 1000);
      return fm.toString(); */
    }

    public long getCdmData2D() {
      return cdmData2D;
    }

    public int getNrecords() {
      return nrecords;
    }

    public boolean isMatch() {
      return nrecords == cdmData2D;
    }

    public int getNvars() {
      return nvars;
    }

    public int getNdups() {
      return ndups;
    }

    public double getNc4SizeM() {
      return ((double) nc4Size) / 1000 / 1000;
    }

    public long getNc4Data2D() {
      return nc4Data2D;
    }

    public double getRatio() {
      long size = f.length();
      return (size == 0) ? 0 : nc4Size / (double) size;
    }

    public long countCdmData2D(NetcdfDataset ncd) throws IOException {
      long result = 0;
      GridDataset ds = new GridDataset(ncd);
      for (GridDatatype grid : ds.getGrids()) {
        int [] shape = grid.getShape();
        int rank = grid.getRank();
        long data2D = 1;
        for (int i=0; i<rank-2; i++)
          data2D *= shape[i];
        result += data2D;
      }
      return result;
    }

    public void countGribData2D(File f) throws IOException {
      String indexFilename = f.getPath() + GribCdmIndex.NCX_SUFFIX;

      FeatureCollectionConfig config = new FeatureCollectionConfig();

      try (GribCollectionImmutable gc = GribCdmIndex.openCdmIndex(indexFilename, config, false, logger)) {
        if (gc == null)
          throw new IOException("Not a grib collection index file");

        for (GribCollectionImmutable.Dataset ds : gc.getDatasets())
          for (GribCollectionImmutable.GroupGC group : ds.getGroups())
            for (GribCollectionImmutable.VariableIndex vi : group.getVariables()) {
              //vi.calcTotalSize();
              //nrecords += vi.nrecords;
              //ndups += vi.ndups;
              nvars++;
            }
      }
    }

    public void countNc4Data2D(File f) {
      String filename = f.getName();

      String nc4Filename = "G:/write/"+filename+".3.grib.nc4";
      File nc4 = new File(nc4Filename);
      if (!nc4.exists()) return;
      nc4Size = nc4.length();

      try (NetcdfDataset ncd = NetcdfDataset.openDataset(nc4Filename)) {
         nc4Data2D = countCdmData2D(ncd);
       } catch (IOException e) {
        System.out.printf("Error opening %s%n", nc4Filename);
      }
    }

  }
}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.servers;

import java.util.*;
import java.io.*;

import opendap.dap.BaseType;

/**
 * Represents a clause containing a simple value. If the value is
 * an constant value such as "2.0", the clause's isConstant() method
 * will return true; if it is a variable of the dataset, isConstant() will
 * return false.
 *
 * @author Joe Wielgosz (joew@cola.iges.org)
 * @see ClauseFactory
 */
public class ValueClause
        extends AbstractClause
        implements SubClause {

    protected BaseType value;
    protected Clause parent;

    /**
     * Creates a new ValueClause.
     *
     * @param value    The BaseType represented by this clause. This can
     *                 be either a BaseType taken from the DDS of a dataset, or a BaseType
     *                 object created to hold a constant value.
     * @param constant Should be set to false if the value parameter is
     *                 from the DDS of a dataset, and true if the value parameter is a
     *                 constant value.
     */
    protected ValueClause(BaseType value,
                          boolean constant) {
        this.value = value;
        this.constant = constant;
        this.defined = constant;
        this.children = new ArrayList();
    }

    /**
     * Returns the BaseType represented by this clause.
     */
    public BaseType getValue() {
        return value;
    }

    /**
     * Returns the BaseType represented by this clause. Equivalent to
     * getValue(), except that
     * calling this method flags this clause as "defined".
     *
     * @throws DAP2ServerSideException Not thrown by this type of clause.
     */
    public BaseType evaluate() {
        defined = true;
        return value;
    }

    public Clause getParent() {
        return parent;
    }

    public void setParent(Clause parent) {
        this.parent = parent;
    }

    /**
     * Prints the original string representation of this clause.
     * For use in debugging.
     */
    public void printConstraint(PrintWriter os)
    {
        if (constant) {
            value.printVal(os, "", false);
        } else {
            value.printConstraint(os);
        }
    }

}


package ucar.nc2.ogc.gml;

import net.opengis.gml.x32.StringOrRefType;
import ucar.nc2.ft.StationTimeSeriesFeature;

/**
 * Created by cwardgar on 2014/02/26.
 */
public abstract class NcStringOrRefType {
    // wml2:Collection/wml2:observationMember/om:OM_Observation/om:featureOfInterest/wml2:MonitoringPoint/
    //         gml:description
    public static StringOrRefType initDescription(StringOrRefType description, StationTimeSeriesFeature stationFeat) {
        // TEXT
        description.setStringValue(stationFeat.getDescription());
        return description;
    }

    private NcStringOrRefType() { }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.bufr.writer;

import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.bufr.Message;
import ucar.nc2.iosp.bufr.BufrIosp2;
import ucar.nc2.iosp.bufr.MessageScanner;
import ucar.nc2.iosp.bufr.BufrNumbers;
import ucar.nc2.util.Indent;
import ucar.nc2.Variable;
import ucar.nc2.*;
import ucar.ma2.*;
import ucar.unidata.io.RandomAccessFile;

import java.io.*;

/**
 * reads a file with BUFR messages in it and prints out the data values.
 *
 * @author caron
 * @since Dec 17, 2009
 */
public class BufrDataProcess {

  private PrintStream out;
  private Indent indent = new Indent(2);
  private boolean showData = false;
  private boolean showMess = false;
  private boolean showFile = true;

  public BufrDataProcess(String filename, OutputStream os, FileFilter ff) throws IOException {
    File f = new File(filename);
    if (!f.exists()) {
      System.out.println(filename + " does not exist");
      return;
    }

    if (f.isDirectory()) {
      Counter gtotal = new Counter();
      int nmess = processAllInDir(f, os, ff, gtotal);
      out.format("%nGrand Total nmess=%d count=%d miss=%d %f %% %n", nmess, gtotal.nvals, gtotal.nmiss, gtotal.percent());
    } else {
      processOneFile(f.getPath(), os, null);
    }
  }

  public int processAllInDir(File dir, OutputStream os, FileFilter ff, Counter gtotal) throws IOException {
    int nmess = 0;

    System.out.println("---------------Reading directory " + dir.getPath());
    File[] allFiles = dir.listFiles();

    if (allFiles == null)
        throw new IOException("Error reading " + dir.getPath());

    for (File f : allFiles) {
      if (f.isDirectory())
        continue;
      if ((ff == null) || ff.accept(f)) {
        nmess += processOneFile(f.getPath(), os, gtotal);
      }
    }

    for (File f : allFiles) {
      if (f.isDirectory())
        nmess += processAllInDir(f, os, ff, gtotal);
    }

    return nmess;
  }

  int processOneFile(String filename, OutputStream os, Counter gtotal) throws IOException {
    out = new PrintStream(os, false, CDM.utf8Charset.name());
    if (showFile) out.format("Process %s%n", filename);
    indent.setIndentLevel(0);
    int nmess;

    try {
      Counter total = new Counter();
      nmess = scanBufrFile(filename, total);
      if (showFile)
        out.format("%nTotal nmess=%d count=%d miss=%d %f %% %n", nmess, total.nvals, total.nmiss, total.percent());
      if (gtotal != null) gtotal.add(total);

    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e.getMessage());
    }

    out.flush();
    return nmess;
  }

  // open the file and extract BUFR messages

  public int scanBufrFile(String filename, Counter total) throws Exception {
    int count = 0;
    try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {

      MessageScanner scan = new MessageScanner(raf);
      while (scan.hasNext()) {
        Message m = scan.next();
        if (m == null) continue;
        try {
          if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader());
          count++;
          Counter counter = new Counter();
          processBufrMessageAsDataset(scan, m, counter);
          if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss);
          total.add(counter);

        } catch (Exception e) {
          System.out.printf("  BARF:%s on %s%n", e.getMessage(), m.getHeader());
          indent.setIndentLevel(0);
        }
      }
    }
    return count;
  }

  // convert one message ino a NetcdfDataset and print data

  private void processBufrMessageAsDataset(MessageScanner scan, Message m, Counter counter) throws Exception {
    byte[] mbytes = scan.getMessageBytes(m);
    NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp");
    Sequence obs = (Sequence) ncfile.findVariable(BufrIosp2.obsRecord);
    StructureDataIterator sdataIter = obs.getStructureIterator(-1);
    processSequence(obs, sdataIter, counter);
  }

  // iterate through the observations

  private void processSequence(Structure s, StructureDataIterator sdataIter, Counter counter) throws IOException {
    indent.incr();
    int count = 0;
    try {
      while (sdataIter.hasNext()) {
        if (showData) out.format("%sSequence %s count=%d%n", indent, s.getShortName(), count++);
        StructureData sdata = sdataIter.next();
        indent.incr();

        for (StructureMembers.Member m : sdata.getMembers()) {
          Variable v = s.findVariable(m.getName());

          if (m.getDataType().isString() || m.getDataType().isNumeric()) {
            processVariable(v, sdata.getArray(m), counter);

          } else if (m.getDataType() == DataType.STRUCTURE) {
            Structure sds = (Structure) v;
            ArrayStructure data = (ArrayStructure) sdata.getArray(m);
            processSequence(sds, data.getStructureDataIterator(), counter);

          } else if (m.getDataType() == DataType.SEQUENCE) {
            Sequence sds = (Sequence) v;
            ArraySequence data = (ArraySequence) sdata.getArray(m);
            processSequence(sds, data.getStructureDataIterator(), counter);
          }
        }
        indent.decr();
      }
    } finally {
      sdataIter.close();
    }

    indent.decr();
  }

  private void processVariable(Variable v, Array mdata, Counter count) throws IOException {
    String name = v.getShortName();
    String units = v.getUnitsString();
    Attribute bwAtt = v.findAttribute("BUFR:bitWidth");
    int bitWidth = bwAtt == null ? 0 : bwAtt.getNumericValue().intValue();

    if (showData) out.format("%svar='%s' units='%s' : ", indent, name, units);

    mdata.resetLocalIterator();
    while (mdata.hasNext()) {
      count.nvals++;
      if (v.getDataType().isUnsigned()) {
        if (isMissingUnsigned(v, mdata, bitWidth)) count.nmiss++;
      } else {
        if (isMissing(v, mdata, bitWidth)) count.nmiss++;
      }
    }
    if (showData) out.format("%n");
  }

  private boolean isMissing(Variable v, Array mdata, int bitWidth) {

    if (v.getDataType().isNumeric()) {
      long val = mdata.nextLong();
      boolean result = BufrNumbers.isMissing(val, bitWidth);
      if (showData) out.format("%d %s,", val, result ? "(miss)" : "");
      return result;
    }

    Object s = mdata.next();
    if (showData) out.format("%s,", s);
    return false;
  }

  private boolean isMissingUnsigned(Variable v, Array mdata, int bitWidth) {
    long val;
    switch (v.getDataType()) {
      case ENUM1:
      case BYTE:
        val = DataType.unsignedByteToShort(mdata.nextByte());
        break;
      case ENUM2:
      case SHORT:
        val = DataType.unsignedShortToInt(mdata.nextShort());
        break;
      case ENUM4:
      case INT:
        val = DataType.unsignedIntToLong(mdata.nextInt());
        break;
      default:
        throw new RuntimeException("illegal datatype " + v.getDataType());
    }

    boolean result = BufrNumbers.isMissing(val, bitWidth);
    if (showData) out.format("%d %s,", val, result ? "(miss)" : "");
    return result;
  }

  private static class Counter {
    int nvals;
    int nmiss;

    void add(Counter c) {
      nvals += c.nvals;
      nmiss += c.nmiss;
    }

    double percent() {
      return 100.0 * nmiss / nvals;
    }
  }

  public static void main(String[] args) throws IOException {
    new BufrDataProcess("C:/data/formats/bufrRoy/", System.out, null);
    //new BufrDataDump("D:/work/michelle/DART.bufr", System.out);
  }

}