/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib2.table;

import ucar.nc2.constants.CDM;
import ucar.nc2.grib.GribTables;
import ucar.nc2.grib.grib2.Grib2Parameter;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;

/**
 * FSL/GSD (center 59)
 * genProcessId 125 = HRRR
 *
 * @author caron
 * @see "http://ruc.noaa.gov/hrrr/GRIB2Table.txt"
 * @since 2/1/12
 */
public class FslHrrrLocalTables extends NcepLocalTables {
  public static final int center_id = 59;

  FslHrrrLocalTables(Grib2TableConfig config) {
    super(config);   // default resource path
    initLocalTable(null);
  }

  @Override
  public String getTablePath(int discipline, int category, int number) {
    if ((category <= 191) && (number <= 191)) return super.getTablePath(discipline, category, number);
    return config.getPath();
  }

  // must override since we are subclassing NcepLocalTables
  @Override
  public List<GribTables.Parameter> getParameters() {
    List<Parameter> result = new ArrayList<>(local.values());
    result.sort(new ParameterSort());
    return result;
  }

  @Override
  public GribTables.Parameter getParameterRaw(int discipline, int category, int number) {
    return local.get(makeParamId(discipline, category, number));
   }

  // LOOK  maybe combine grib1, grib2 and bufr ??
  @Override
  public String getSubCenterName(int center, int subcenter) {

    switch (subcenter) {
      case 0:
        return null;
      case 1:
        return "FSL/FRD Regional Analysis and Prediction Branch";
      case 2:
        return "FSL/FRD Local Analysis and Prediction Branch";
    }
    return super.getSubCenterName(center, subcenter);
  }

  @Override
  public String getGeneratingProcessName(int genProcess) {
    switch (genProcess) {
      case 103:
        return "ExREF";
      case 105:
        return "RAP/ RUC";
      case 106:
        return "Developmental Testbed Center Winter Field Experiment, WRF-ARW";
      case 112:
        return "Developmental Testbed Center Winter Field Experiment, WRF-NMM";
      case 116:
        return "Flow-following Finite-volume Icosahedral Model (FIM)";
      case 125:
        return "High-Resolution Rapid Refresh";
      default:
        return null;
    }
  }

  @Override
  public String getLevelNameShort(int id) {
    if (id == 200) {
      return "Entire_atmosphere";
    }
    return super.getLevelNameShort(id);
  }

  @Override

  public String getLevelName(int id) {
    if (id == 200) {
      return "Entire atmosphere layer";
    }
    return super.getLevelName(id);
  }

  public String getStatisticNameShort(int id) {
    if (id == 255) {
      return "Interval";
    }
    return super.getStatisticNameShort(id);
  }

  private void initLocalTable(Formatter f) {
    local = readCsv(config.getPath(), f);
  }

  // debugging
  @Override
  public void lookForProblems(Formatter f) {
    initLocalTable(f);
  }

  private Map<Integer, Grib2Parameter> readCsv(String resourcePath, Formatter f) {
    boolean header = true;
    Map<Integer, Grib2Parameter> result = new HashMap<>(100);

    ClassLoader cl = getClass().getClassLoader();
    try (InputStream is = cl.getResourceAsStream(resourcePath)) {
      if (is == null) throw new IllegalStateException("Cant find " + resourcePath);
      try (BufferedReader br = new BufferedReader(new InputStreamReader(is, CDM.utf8Charset))) {
        HashMap<String, Grib2Parameter> names = new HashMap<>(200);

        while (true) {
          String line = br.readLine();
          if (line == null) {
            break;
          }
          if (line.startsWith("Record")) {
            break;
          }
        }

        while (true) {
          String line = br.readLine();
          if (line == null) {
            break;
          }
          if ((line.length() == 0) || line.startsWith("#")) {
            continue;
          }
          String[] flds = line.split(",");

          //RecordNumber,	TableNumber,	DisciplineNumber,	CategoryNumber,	ParameterNumber,	WGrib2Name,	NCLName,				FieldType,			Description,													Units,
          String recordNumber = flds[0].trim();
          int tableNumber = Integer.parseInt(flds[1].trim());
          int disciplineNumber = Integer.parseInt(flds[2].trim());
          int categoryNumber = Integer.parseInt(flds[3].trim());
          int parameterNumber = Integer.parseInt(flds[4].trim());

          String WGrib2Name = flds[5].trim();
          String NCLName = flds[6].trim();
          String FieldType = flds[7].trim();
          String Description = flds[8].trim();
          String Units = flds[9].trim();
          if (f != null) {
            f.format("%3s %3d %3d %3d %3d %-10s %-25s %-30s %-100s %-20s%n", recordNumber,
                tableNumber, disciplineNumber, categoryNumber, parameterNumber,
                WGrib2Name, NCLName, FieldType, Description, Units);
          }

          String name = !WGrib2Name.equals("var") ? WGrib2Name : FieldType;
          Grib2Parameter s = new Grib2Parameter(disciplineNumber, categoryNumber, parameterNumber,
              name, Units, null, Description);
          // s.desc = Description;
          result.put(makeParamId(disciplineNumber, categoryNumber, parameterNumber), s);
          if (f != null) {
            f.format(" %s%n", s);
          }
          if (categoryNumber > 191 || parameterNumber > 191) {
            Grib2Parameter dup = names.get(s.getName());
            if (dup != null && f != null) {
              if (header) {
                f.format("Problems in table %s %n", resourcePath);
              }
              header = false;
              f.format(" DUPLICATE NAME %s and %s (%s)%n", s.getId(), dup.getId(), s.getName());
            }
          }
          names.put(s.getName(), s);
        }
      }

    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }

    return result;
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.time;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.joda.time.DurationFieldType;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import ucar.nc2.units.TimeDuration;
import ucar.unidata.util.StringUtil2;

import javax.annotation.concurrent.Immutable;

/**
 * A CalendarPeriod is a logical duration of time, it requires a Calendar to convert to an actual duration of time.
 * A CalendarField is expressed as {integer x Field}.
 *
 * Design follows joda Period class.
 * @author caron
 * @since 3/30/11
 */
@Immutable
public class CalendarPeriod {
  private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(CalendarPeriod.class);

  private static final Cache<CalendarPeriod, CalendarPeriod> cache = CacheBuilder.newBuilder()
                .maximumSize(100)  // limit cache size....
                .build();

  public static final CalendarPeriod Hour = CalendarPeriod.of(1, Field.Hour);

  public enum Field {
    Millisec(PeriodType.millis()), Second(PeriodType.seconds()), Minute(PeriodType.minutes()), Hour(PeriodType.hours()),
    Day(PeriodType.days()), Month(PeriodType.months()), Year(PeriodType.years());

    PeriodType p;
    Field(PeriodType p) {
      this.p = p;
    }
  }

  /**
   * Convert a period string into a CalendarPeriod.Field.
   * @param udunit period string
   * @return CalendarPeriod.Field enum
   * @throws IllegalArgumentException if not valid format
   */
  public static CalendarPeriod.Field fromUnitString(String udunit) {
    udunit = udunit.trim();
    udunit = udunit.toLowerCase();

    if (udunit.equals("s")) return Field.Second;
    if (udunit.equals("ms")) return Field.Millisec;

      // eliminate plurals
    if (udunit.endsWith("s")) udunit = udunit.substring(0, udunit.length()-1);

    switch (udunit) {
      case "second":
      case "sec":
        return Field.Second;
      case "millisecond":
      case "millisec":
      case "msec":
        return Field.Millisec;
      case "minute":
      case "min":
        return Field.Minute;
      case "hour":
      case "hr":
      case "h":
        return Field.Hour;
      case "day":
      case "d":
        return Field.Day;
      case "month":
      case "mon":
        return Field.Month;
      case "year":
      case "yr":
        return Field.Year;
      default:
        throw new IllegalArgumentException("cant convert " + udunit + " to CalendarPeriod");
    }
  }

  // minimize memory use by interning. wacko shit in GribPartitionBuilder TimeCoordinate, whoduhthunk?
  public static CalendarPeriod of(int value, Field field) {
    CalendarPeriod want = new CalendarPeriod(value, field);
    if (cache == null) return want;
    CalendarPeriod got = cache.getIfPresent(want);
    if (got != null) return got;
    cache.put(want, want);
    return want;
  }

  /**
   * Convert a udunit period string into a CalendarPeriod
   * @param udunit period string : "[val] unit"
   * @return CalendarPeriod or null if illegal
   */
  public static CalendarPeriod of(String udunit) {
    int value;
    String units;

    String[] split = StringUtil2.splitString(udunit);
    if (split.length == 1) {
      value = 1;
      units =  split[0];

    } else if (split.length == 2) {
      try {
        value = Integer.parseInt(split[0]);
      } catch (Throwable t) {
        return null;
      }
      units =  split[1];
    } else
      return null;

    CalendarPeriod.Field unit = CalendarPeriod.fromUnitString(units);
    return CalendarPeriod.of(value, unit);
  }

  public static CalendarPeriod of(TimeDuration td) {
    CalendarPeriod.Field unit = CalendarPeriod.fromUnitString(td.getTimeUnit().getUnitString());
    return CalendarPeriod.of( (int) td.getValue(), unit);
  }

  ////////////////////////
  // the common case is a single field
  private final int value;
  private final Field field;

  private CalendarPeriod (int value, Field field) {
    this.value = value;
    this.field = field;
  }

  /**
   * Multiply the period by an integer
   * @param value multiply by this
   * @return new period
   */
  public CalendarPeriod multiply(int value) {
    return CalendarPeriod.of(this.value * value, this.field);
  }

  public int getValue() {
    return value;
  }

  public Field getField() {
    return field;
  }

  /**
   * Subtract two dates, return difference in units of this period.
   * If not even, will round down and log a warning
   * @param start start date
   * @param end   end date
   * @return  difference in units of this period
   */
  public int subtract(CalendarDate start, CalendarDate end) {
    long diff = end.getDifferenceInMsecs(start);
    int thislen = millisecs();
    if ((diff % thislen != 0))
      log.warn("roundoff error");
    return (int) (diff / thislen);
  }

  /**
   * Get the conversion factor of the other CalendarPeriod to this one
   * @param from convert from this
   * @return conversion factor, so that getConvertFactor(from) * from = this
   */
  public double getConvertFactor(CalendarPeriod from) {
    if (field == CalendarPeriod.Field.Month || field == CalendarPeriod.Field.Year) {
      log.warn(" CalendarDate.convert on Month or Year");
    }

    return (double) from.millisecs() / millisecs();
  }

  /**
   * Get the duration in milliseconds                                               -+
   * @return the duration in seconds
   * @deprecated dont use because these are fixed length and thus approximate.
   */
  public double getValueInMillisecs() {
     if (field == CalendarPeriod.Field.Month)
       return 30.0 * 24.0 * 60.0 * 60.0 * 1000.0 * value;
     else if (field == CalendarPeriod.Field.Year)
       return 365.0 * 24.0 * 60.0 * 60.0 * 1000.0 * value;
     else return millisecs();
   }

  private int millisecs() {
     if (field == CalendarPeriod.Field.Millisec)
       return value;
     else if (field == CalendarPeriod.Field.Second)
       return 1000 * value;
     else if (field == CalendarPeriod.Field.Minute)
       return 60 * 1000 * value;
     else if (field == CalendarPeriod.Field.Hour)
       return 60 * 60 * 1000 * value;
     else if (field == CalendarPeriod.Field.Day)
       return 24 * 60 * 60 * 1000 * value;

     else throw new IllegalStateException("Illegal Field = "+field);
   }

   /* LOOK from old TimeCoord code, which is better ??
  public static int getOffset(CalendarDate refDate, CalendarDate cd, CalendarPeriod timeUnit) {
    long msecs = cd.getDifferenceInMsecs(refDate);
    return (int) Math.round(msecs / timeUnit.getValueInMillisecs());
  }
  */

  // offset from start to end, in these units
  // start + offset = end
  public int getOffset(CalendarDate start, CalendarDate end) {
    if (start.equals(end)) return 0;
    long start_millis = start.getDateTime().getMillis();
    long end_millis = end.getDateTime().getMillis();

    // 5 second slop
    Period p;
    if (start_millis < end_millis)
      p = new Period(start_millis, end_millis + 5000, getPeriodType());
    else
      p = new Period(start_millis+5000, end_millis, getPeriodType());

    return p.get(getDurationFieldType());
  }

  PeriodType getPeriodType() {
    return getField().p;
  }

  DurationFieldType getDurationFieldType() {
    return getField().p.getFieldType(0);
  }

  @Override
  public String toString() {
    return value + " " + field;
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;

    CalendarPeriod that = (CalendarPeriod) o;

    if (value != that.value) return false;
    if (field != that.field) return false;

    return true;
  }

  @Override
  public int hashCode() {
    int result = value;
    result = 31 * result + (field != null ? field.hashCode() : 0);
    return result;
  }

  public static void main(String[] args) {
    CalendarPeriod cp = CalendarPeriod.of(1, Field.Day);
    CalendarDate start =  CalendarDate.parseUdunits(null, "3 days since 1970-01-01 12:00");
    CalendarDate end =  CalendarDate.parseUdunits(null, "6 days since 1970-01-01 12:00");
    int offset = cp.getOffset(start, end);
    System.out.printf("offset=%d%n", offset);
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.gempak;

import ucar.nc2.iosp.grid.*;

import ucar.nc2.*;
import ucar.nc2.iosp.grid.GridIndexToNC;
import ucar.nc2.iosp.grid.GridServiceProvider;

import ucar.nc2.util.CancelTask;
import ucar.unidata.io.RandomAccessFile;
import java.io.IOException;

/**
 * An IOSP for GEMPAK Grid data
 *
 * @author Don Murray
 */
public class GempakGridServiceProvider extends GridServiceProvider {

  /**
   * GEMPAK file reader
   */
  protected GempakGridReader gemreader;

  /**
   * Reread the file on a sync
   */
  public static boolean extendIndex = true;  // check if index needs to be extended

  /**
   * Is this a valid file?
   *
   * @param raf RandomAccessFile to check
   * @return true if a valid GEMPAK grid file
   * @throws IOException problem reading file
   */
  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    try {
      gemreader = new GempakGridReader(raf.getLocation());
      return gemreader.init(raf, false);
    } catch (Exception ioe) {
      return false;
    }
  }

  public String getFileTypeId() {
    return "GempakGrid";
  }

  public String getFileTypeDescription() {
    return "GEMPAK Gridded Data";
  }

  /**
   * Open the service provider for reading.
   *
   * @param raf        file to read from
   * @param ncfile     netCDF file we are writing to (memory)
   * @param cancelTask task for cancelling
   * @throws IOException problem reading file
   */
  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    super.open(raf, ncfile, cancelTask);
    // debugProj = true;
    long start = System.currentTimeMillis();
    if (gemreader == null) {
      gemreader = new GempakGridReader(raf.getLocation());
    }
    initTables();
    gemreader.init(raf, true);
    GridIndex index = gemreader.getGridIndex();
    open(index, cancelTask);
    if (debugOpen) {
      System.out.println(" GridServiceProvider.open "
              + ncfile.getLocation() + " took "
              + (System.currentTimeMillis() - start));
    }
  }

  /**
   * Open the index and create the netCDF file from that
   *
   * @param index      GridIndex to use
   * @param cancelTask cancel task
   * @throws IOException problem reading the file
   */
  protected void open(GridIndex index, CancelTask cancelTask) throws IOException {
    GempakLookup lookup = new GempakLookup((GempakGridRecord) index.getGridRecords().get(0));
    GridIndexToNC delegate = new GridIndexToNC(index.filename);
    //delegate.setUseDescriptionForVariableName(false);
    delegate.open(index, lookup, 4, ncfile, cancelTask);
    ncfile.finish();
  }

  /**
   * Sync the file
   *
   * @return true if needed to sync
   * @throws IOException problem synching the file
   */
  public boolean sync() throws IOException {
    if ((gemreader.getInitFileSize() < raf.length()) && extendIndex) {
      gemreader.init(true);
      GridIndex index = gemreader.getGridIndex();
      // reconstruct the ncfile objects
      ncfile.empty();
      open(index, null);
      return true;
    }
    return false;
  }

  /**
   * Read the data for this GridRecord
   *
   * @param gr grid identifier
   * @return the data (or null)
   * @throws IOException problem reading the data
   */
  protected float[] _readData(GridRecord gr) throws IOException {
    return gemreader.readGrid( gr);
  }

  /**
   * Initialize the parameter tables.
   */
  private void initTables() {
    try {
      GempakGridParameterTable.addParameters(
              "resources/nj22/tables/gempak/wmogrib3.tbl");
      GempakGridParameterTable.addParameters(
              "resources/nj22/tables/gempak/ncepgrib2.tbl");
    } catch (Exception e) {
      System.out.println("unable to init tables");
    }
  }

  /**
   * Extend the list of grid
   *
   * @param b true to reread the grid on a sync
   */
  public static void setExtendIndex(boolean b) {
    extendIndex = b;
  }
}


/* Copyright */
package thredds.server.catalog;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.client.catalog.CatalogRef;
import thredds.client.catalog.Dataset;
import thredds.server.catalog.tracker.DataRootExt;
import thredds.server.catalog.tracker.DataRootTracker;
import ucar.nc2.util.AliasTranslator;
import ucar.unidata.util.StringUtil2;

import javax.annotation.Nonnull;
import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Find the dataRoot path from the request, by getting the longest match.
 * Use a TreeSet for minimum in-memory use.
 * Keep the objects in a seperate map that could be off-heap and persistent.
 *
 * @author caron
 * @since 4/1/2015
 */
public class DataRootPathMatcher {
  static private final Logger logger = LoggerFactory.getLogger(DataRootPathMatcher.class);
  static private org.slf4j.Logger logCatalogInit = org.slf4j.LoggerFactory.getLogger("catalogInit");
  static private final boolean debug = false;
  static private final String ERROR = "*** ERROR: ";
  static private boolean skipTestDataDir = true;

  private static class PathComparator implements Comparator<String> {
    public int compare(String s1, String s2) {
      int compare = s2.compareTo( s1); // reverse sort
      if (debug) System.out.println(" compare "+s1+" to "+s2+" = "+compare);
      return compare;
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////////

  private ConfigCatalogCache ccc;
  private DataRootTracker tracker;

  private final TreeSet<String> treeSet = new TreeSet<>( new PathComparator());    // this should be in-memory for speed
  private final Map<String, DataRootExt> map = new HashMap<>();         // this could be turned into an off-heap cache if needed, with persistence.

  public DataRootPathMatcher(ConfigCatalogCache ccc, DataRootTracker tracker) {
    this.ccc = ccc;
    this.tracker = tracker;

    for (DataRootExt dre : tracker.getDataRoots()) {
      put(dre);
    }
  }

  /**
   * Add a dataRootExt to in-memory tree.
   * @return true if not already exist
   */
  private boolean put(DataRootExt dateRootExt) {
    map.put(dateRootExt.getPath(), dateRootExt);
    return treeSet.add(dateRootExt.getPath());
  }

  /**
   * See if this object already exists in the collection
   * @param path find object that has this key
   * @return true if already contains the key
   */
  public boolean contains(String  path) {
    return treeSet.contains(path);
  }

  public DataRootExt get(String  path) {
    return map.get(path);
  }

  /**
   * Get an iterator over the dataRoot keys and values. debug
   * @return iterator
   */
  public Set<Map.Entry<String, DataRootExt>> getValues() {
    return map.entrySet();
  }

  /**
   * Find the longest path match.
   * @param reqPath find object with longest match where reqPath.startsWith( key)
   * @return the value whose key is the longest that matches path, or null if none
   */
  public String findLongestPathMatch( String reqPath) {
    SortedSet<String> tail = treeSet.tailSet(reqPath);
    if (tail.isEmpty()) return null;
    String after = tail.first();
    if (reqPath.startsWith( after)) // common case
      return tail.first();

    // have to check more, until no common starting chars
    for (String key : tail) {
      if (reqPath.startsWith(key))
        return key;

      // terminate when there's no match at all.
      if (StringUtil2.match(reqPath, key) == 0)
        break;
    }

    return null;
  }

  /**
   * Find the longest DataRoot match.
   * @param reqPath find object with longest match where reqPath.startsWith( key)
   * @return the value whose key is the longest that matches path, or null if none
   */
  public DataRoot findDataRoot( String reqPath) {
    String path =  findLongestPathMatch(reqPath);
    if (path == null) return null;
    DataRootExt dataRootExt = map.get(path);
    if (dataRootExt == null) {
      logger.error("DataRootPathMatcher found path {} but not in map", path);
      return null;
    }
    return convert2DataRoot(dataRootExt);
  }

  // convert a dataRootExt to a dataRoot
  public @Nonnull DataRoot convert2DataRoot(DataRootExt dataRootExt) {
    DataRoot dataRoot = dataRootExt.getDataRoot();
    if (dataRoot != null) return dataRoot;

    // otherwise must read the catalog that its in
    dataRoot = readDataRootFromCatalog(dataRootExt);
    dataRootExt.setDataRoot(dataRoot);
    return dataRoot;
  }

  private @Nonnull DataRoot readDataRootFromCatalog( DataRootExt dataRootExt) {
    try {
      ConfigCatalog cat = ccc.get(dataRootExt.getCatLocation());
      extractDataRoots(dataRootExt.getCatLocation(), cat.getDatasetsLocal(), false, null);  // will create a new DataRootExt and replace this one in the map
      DataRootExt dataRootExtNew = map.get(dataRootExt.getPath());
      if (null == dataRootExtNew) {
        logger.error("Reading catalog " + dataRootExt.getCatLocation() + " failed to find dataRoot path=" + dataRootExt.getPath());
        throw new IllegalStateException("Reading catalog " + dataRootExt.getCatLocation() + " failed to find dataRoot path=" + dataRootExt.getPath());
      }
      return dataRootExtNew.getDataRoot();

    } catch (IOException e) {
      throw new IllegalStateException("Error Reading catalog " + dataRootExt.getCatLocation(), e);
    }
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
  // building up the data roots after reading catalogs

  /**
   * Finds datasetScan, datasetFmrc
   * Look for duplicate Ids (give message).
   * Dont follow catRefs.
   *
   * @param dsList the list of Dataset
   */
  public void extractDataRoots(String catalogRelPath, List<Dataset> dsList, boolean checkDups, Map<String, String> idMap) {

    for (Dataset dataset : dsList) {
      if (dataset instanceof DatasetScan) {
        DatasetScan ds = (DatasetScan) dataset;
        addRoot(ds, catalogRelPath, checkDups);

      } else if (dataset instanceof FeatureCollectionRef) {
        FeatureCollectionRef fc = (FeatureCollectionRef) dataset;
        addRoot(fc, catalogRelPath, checkDups);

        if (idMap != null) {
          String catWithSameFc = idMap.get(fc.getCollectionName());
          if (catWithSameFc != null)
            logCatalogInit.warn("*** ERROR: Duplicate featureCollection name {} in catalogs '{}' and '{}'", fc.getCollectionName(), catalogRelPath, catWithSameFc);
          else
            idMap.put(fc.getCollectionName(), catalogRelPath);
        }

      } else if (dataset instanceof CatalogScan) {
        CatalogScan catScan = (CatalogScan) dataset;
        addRoot(catScan, catalogRelPath, checkDups);
      }

      if (!(dataset instanceof CatalogRef)) {
        // recurse
        extractDataRoots(catalogRelPath, dataset.getDatasetsLocal(), checkDups, idMap);
      }
    }
  }

  private boolean addRoot(DatasetScan dscan, String catalogRelPath, boolean checkDups) {
    String path = dscan.getPath();

    if (path == null) {
      logCatalogInit.error(ERROR + "DatasetScan '" + dscan.getName() + "' missing the path attribute.");
      return false;
    }

    if (checkDups) {
      DataRootExt dre = get(path); // check for duplicates
      if (dre != null) {
        logCatalogInit.error(ERROR + "DatasetScan trying to add duplicate dataRoot =<" + path + ">  already mapped to directory= <" + dre.getDirLocation() + ">" +
                " wanted to map to =<" + dscan.getScanLocation() + "> in catalog " + catalogRelPath);
        return false;
      }
    }

    // check for existence
    File file = new File(dscan.getScanLocation());
    if (!skipTestDataDir && !file.exists()) {
      logCatalogInit.error(ERROR + "DatasetScan path ='" + path + "' directory= <" + dscan.getScanLocation() + "> does not exist");
      return false;
    }

    // add it
    putRoot(new DataRoot(dscan), catalogRelPath);
    logCatalogInit.debug(" added rootPath=<" + path + ">  for directory= <" + dscan.getScanLocation() + ">");
    return true;
  }

  private boolean addRoot(FeatureCollectionRef fc, String catalogRelPath, boolean checkDups) {
    String path = fc.getPath();

    if (path == null) {
      logCatalogInit.error(ERROR + "DatasetScan '" + fc.getName() + "' missing the path attribute.");
      return false;
    }

    if (checkDups) {
      DataRootExt dre = get(path); // check for duplicates
      if (dre != null) {
        logCatalogInit.error(ERROR + "FeatureCollection trying to add duplicate dataRoot =<" + path + ">  already mapped to directory= <" + dre.getDirLocation() + ">" +
                " wanted to map to =<" + fc.getTopDirectoryLocation() + "> in catalog " + catalogRelPath);
        return false;
      }
    }

    // check for existence
    File file = new File(fc.getTopDirectoryLocation());
    if (!skipTestDataDir && !file.exists()) {
      logCatalogInit.error(ERROR + "FeatureCollection path ='" + path + "' directory= <" + fc.getTopDirectoryLocation() + "> does not exist");
      return false;
    }

    // add it
    putRoot(new DataRoot(fc), catalogRelPath);
    logCatalogInit.debug(" added rootPath=<" + path + ">  for feature collection= <" + fc.getName() + ">");
    return true;
  }

  public boolean addRoot(DatasetRootConfig config, String catalogRelPath, boolean checkDups) {
    String path = config.getPath();

    if (path == null) {
      logCatalogInit.error(ERROR + "DatasetRoot '" + config + "' missing the path attribute.");
      return false;
    }

    if (checkDups) {
      DataRootExt dre = get(path); // check for duplicates
      if (dre != null) {
        logCatalogInit.error(ERROR + "DatasetRoot trying to add duplicate dataRoot =<" + path + ">  already mapped to directory= <" + dre.getDirLocation() + ">" +
                " wanted to map to =<" + config.getLocation() + "> in catalog " + catalogRelPath);
        return false;
      }
    }

    // translate and check for existence
    String location = AliasTranslator.translateAlias(config.getLocation());
    File file = new File(location);
    if (!skipTestDataDir && !file.exists()) {
      logCatalogInit.error(ERROR + "DatasetRootConfig path ='" + path + "' directory= <" + location + "> does not exist");
      return false;
    }

    // add it
    putRoot(new DataRoot(config.getPath(), location, null), catalogRelPath); // LOOK would be easy to add a restrict here
    logCatalogInit.debug(" added rootPath=<" + path + ">  for DatasetRootConfig location= <" + location + ">");
    return true;
  }

  private boolean addRoot(CatalogScan catScan, String catalogRelPath, boolean checkDups) {
    String path = catScan.getPath();

    if (path == null) {
      logCatalogInit.error(ERROR + "CatalogScan '" + catScan + "' missing the path attribute.");
      return false;
    }

    if (checkDups) {
      DataRootExt dre = get(path); // check for duplicates
      if (dre != null) {
        logCatalogInit.error(ERROR + "CatalogScan trying to add duplicate dataRoot =<" + path + ">  already mapped to directory= <" + dre.getDirLocation() + ">" +
                " wanted to map to =<" + catScan.getLocation() + "> in catalog " + catalogRelPath);
        return false;
      }
    }

    // translate and check for existence
    String location = AliasTranslator.translateAlias(catScan.getLocation());
    File file = new File(location);
    if (!skipTestDataDir && !file.exists()) {
      logCatalogInit.error(ERROR + "CatalogScan path ='" + path + "' directory= <" + location + "> does not exist");
      return false;
    }

    // add it
    putRoot(new DataRoot(catScan), catalogRelPath);
    logCatalogInit.debug(" added rootPath=<" + path + ">  for CatalogScan location= <" + location + ">");
    return true;
  }

  private void putRoot(DataRoot droot, String catalogRelPath) {
    DataRootExt drootExt = new DataRootExt(droot, catalogRelPath);
    put(drootExt);
    tracker.trackDataRoot(drootExt);
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.server.ncss.params;

import thredds.server.ncss.exception.NcssException;
import thredds.server.ncss.validation.TimeParamsConstraint;
import thredds.server.ncss.validation.VarParamConstraint;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.ft.point.collection.UpdateableCollection;
import ucar.nc2.time.Calendar;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateFormatter;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;
import ucar.nc2.units.TimeDuration;
import ucar.unidata.geoloc.LatLonPointImpl;
import ucar.unidata.geoloc.LatLonRect;

import java.io.IOException;
import java.text.ParseException;
import java.util.Formatter;
import java.util.List;

/**
 * Ncss Parameters superclass, have common parameters for grid and point
 *
 * @author caron
 * @since 10/5/13
 */

@TimeParamsConstraint
public class NcssParamsBean {

  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger("featureCollectionScan");

  protected String accept;

  @VarParamConstraint
  protected List<String> var;

  protected String time_start;

  protected String time_end;

  protected String time_duration;

  protected String time;

  protected String temporal;  // legacy, use time="all"

  protected String time_window;        // a time duration; point data only

  protected Double north;

  protected Double south;

  protected Double east;

  protected Double west;

  protected Double latitude;

  protected Double longitude;

  public NcssParamsBean() {}

  public NcssParamsBean(NcssParamsBean from) {
    this.accept = from.accept;
    this.var = from.var;
    this.time_start = from.time_start;
    this.time_end = from.time_end;
    this.time_duration = from.time_duration;
    this.time = from.time;
    this.temporal = from.temporal;
    this.north = from.north;
    this.south = from.south;
    this.east = from.east;
    this.west = from.west;
    this.latitude = from.latitude;
    this.longitude = from.longitude;
    this.date = from.date;
    this.dateRange = from.dateRange;
  }

  /////////////////////////////////////////////////////
  // Bean set/getters, needed by Spring

  public String getAccept() {
    return accept;
  }

  public void setAccept(String accept) {
    this.accept = accept;
  }

  public List<String> getVar() {
    return var;
  }

  public void setVar(List<String> var) {
    this.var = var;
  }

  // time - do not use directly, use time methods below
  public String getTime_start() {
    return time_start;
  }

  public void setTime_start(String time_start) {
    this.time_start = time_start;
  }

  public String getTime_end() {
    return time_end;
  }

  public void setTime_end(String time_end) {
    this.time_end = time_end;
  }

  public String getTime_duration() {
    return time_duration;
  }

  public void setTime_duration(String time_duration) {
    this.time_duration = time_duration;
  }

  public String getTime() {
    return time;
  }
  public void setTime(String time) {
    this.time = time;
  }

  // deprecated
  public String getTemporal() {
    return temporal;
  }
  public void setTemporal(String temporal) {
    this.temporal = temporal;
  }

  public String getTime_window() {
    return time_window;
  }

  public void setTime_window(String time_window) {
    this.time_window = time_window;
  }

  // latlon
  public Double getNorth() {
    return north;
  }

  public void setNorth(Double north) {
    this.north = north;
  }

  public Double getSouth() {
    return south;
  }

  public void setSouth(Double south) {
    this.south = south;
  }

  public Double getEast() {
    return east;
  }

  public void setEast(Double east) {
    this.east = east;
  }

  public Double getWest() {
    return west;
  }

  public void setWest(Double west) {
    this.west = west;
  }

  public Double getLatitude() {
    return latitude;
  }

  public void setLatitude(Double latitude) {
    this.latitude = latitude;
  }

  public Double getLongitude() {
    return longitude;
  }

  public void setLongitude(Double longitude) {
    this.longitude = longitude;
  }

  public boolean hasLatLonPoint() {
    return latitude != null && longitude != null;
  }

  public boolean hasLatLonBB() {
    return east != null && west != null && north != null && south != null;
  }

  public LatLonRect getLatLonBoundingBox() {
    if (!hasLatLonBB()) {
      return null;
    } else {
      double width = getEast() - getWest();
      double height = getNorth() - getSouth();
      return new LatLonRect(new LatLonPointImpl(getSouth(), getWest()), height, width);
    }
  }

  //////////////////////////////////////////////////////////
  // time_window is meant to be used with time=present. When time=present it returns the closest time to current in the dataset
  // but if the dataset does not have up to date data that could be really far from the current time and most likely useless (esp for observation data).
  // time_window tells the server give me the data if it's within this period otherwise don't bother. time_window must be a valid W3C time duration or udunit time

  protected TimeDuration timeWindow;
  public TimeDuration getTimeWindow() {
    return timeWindow;
  }

  public void setTimeWindow(TimeDuration timeWindow) {
    this.timeWindow = timeWindow;
  }

  //////////////////////////////////////////////////
  // time methods

  // problem is we dont know the Calendar until we open the dataset
  protected CalendarDate date;
  protected CalendarDateRange dateRange;

  public void setDate(CalendarDate date) {
    this.date = date;
  }

  public void setDateRange(CalendarDateRange dateRange) {
    this.dateRange = dateRange;
  }

  public boolean isAllTimes() {
    return (temporal != null && temporal.equalsIgnoreCase("all")) || (time != null && time.equalsIgnoreCase("all"));
  }

  public boolean isPresentTime() {
    return (time != null && time.equalsIgnoreCase("present"));
  }

  // return requested CalendarDateRange.
  public CalendarDateRange getCalendarDateRange(Calendar cal) {
    if (dateRange == null) return null;
    if (cal.equals(Calendar.getDefault())) return dateRange;

    // otherwise must reparse
    return makeCalendarDateRange(cal);
  }

  public CalendarDateRange makeCalendarDateRange(Calendar cal) {
    try {
      // this handles "present"
      DateRange dr = new DateRange(new DateType(time_start, null, null, cal), new DateType(time_end, null, null, cal), new TimeDuration(time_duration), null);
      return CalendarDateRange.of(dr.getStart().getCalendarDate(), dr.getEnd().getCalendarDate());
    } catch (ParseException pe) {
      return null; // ??
    }
  }

  public CalendarDate getRequestedDate(Calendar cal) {
    if (date == null) return null;
    if (cal.equals(Calendar.getDefault())) return date;

     // otherwise must reparse
    return CalendarDateFormatter.isoStringToCalendarDate(cal, getTime());
  }

  public boolean intersectsTime(CalendarDateRange have, Formatter errs) {
    if (have == null) return true;
    Calendar dataCal = have.getStart().getCalendar(); // use the same calendar as the dataset

    CalendarDateRange want = getCalendarDateRange(dataCal);
    if (want != null) {
      if (have.intersects(want)) {
        return true;
      } else {
        errs.format("Requested time range %s does not intersect actual time range %s", want, have);
        return false;
      }
    }

    CalendarDate wantTime = getRequestedDate(dataCal);
    if (wantTime == null) return true;
    if (!have.includes(wantTime)) {
      errs.format("Requested time %s does not intersect actual time range %s", wantTime, have);
      return false;
    }
    return true;
  }

  public TimeDuration parseTimeDuration() throws NcssException {
    if (getTime_duration() == null) return null;
    try {
      return TimeDuration.parseW3CDuration(getTime_duration());
    } catch (ParseException e) {
      throw new NcssException("invalid time duration");
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.grib.coord;

import java.util.*;

/**
 * Create shared coordinate variables across the variables in the same group.
 * Use Coordinate.equals(), hashCode() to find unique coordinate variables.
 *
 * Used by GribPartitionBuilder.makeBest(), for all coordinates not CoordinateTime2D and not CoordinateRuntime.
 * Best coordinates are translated into equivilent 2D coordinate, thus we dont need seperate CoordinateND.
 *
 * @author caron
 * @since 12/10/13
 */
public class CoordinateSharerBest {
  private Map<Coordinate, Coordinate> runtimeBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> timeBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> timeIntvBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> vertBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> ensBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> time2DBuilders = new HashMap<>();
  private Map<Coordinate, Coordinate> swap = new HashMap<>();

  private Map<Coordinate, Integer> indexMap;

  public void addCoordinates(List<Coordinate> coords) {
    for (Coordinate coord : coords) addCoordinate(coord);
  }

  public void addCoordinate(Coordinate coord) {
    Coordinate already;
    switch (coord.getType()) {
      case runtime:
        already = runtimeBuilders.get(coord);
        if (already == null)
          runtimeBuilders.put(coord, coord);
        else
          swap.put(coord, already);  // keep track of substitutes
        break;

      case time:
        already = timeBuilders.get(coord);
        if (already == null)
          timeBuilders.put(coord, coord);
        else
          swap.put(coord, already);
        break;

      case timeIntv:
        already = timeIntvBuilders.get(coord);
        if (already == null)
          timeIntvBuilders.put(coord, coord);
        else
          swap.put(coord, already);
        break;

      case time2D:
        already = time2DBuilders.get(coord);
        if (already == null)
          time2DBuilders.put(coord, coord);
        else
          swap.put(coord, already);
        break;

      case vert:
        already = vertBuilders.get(coord);
        if (already == null)
          vertBuilders.put(coord, coord);
        else
          swap.put(coord, already);
        break;

      case ens:
        already = ensBuilders.get(coord);
        if (already == null)
          ensBuilders.put(coord, coord);
        else
          swap.put(coord, already);
        break;
    }
  }

   public List<Coordinate> finish() {
     // results
     List<Coordinate> unionCoords = new ArrayList<>(20);
     unionCoords.addAll(runtimeBuilders.keySet());
     unionCoords.addAll(time2DBuilders.keySet());
     unionCoords.addAll(timeBuilders.keySet());
     unionCoords.addAll(timeIntvBuilders.keySet());
     unionCoords.addAll(vertBuilders.keySet());
     unionCoords.addAll(ensBuilders.keySet());

     indexMap = new HashMap<>();
     for (int i = 0; i < unionCoords.size(); i++) {
       indexMap.put(unionCoords.get(i), i);
     }

     return unionCoords;
   }

     // redo the variables against the shared coordinates
    public List<Integer> reindex(List<Coordinate> coords) {
     List<Integer> result = new ArrayList<>();
     for (Coordinate coord : coords) {
       Coordinate sub = swap.get(coord);
       Coordinate use = (sub == null) ? coord : sub;
       Integer idx = indexMap.get(use); // index into unionCoords
       if (idx == null) {
         throw new IllegalStateException();
       }
       result.add(idx);
     }
     return result;
   }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2;

import ucar.ma2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.IOServiceProviderWriter;
import ucar.nc2.iosp.hdf5.H5header;
import ucar.nc2.iosp.netcdf3.N3header;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.nc2.iosp.netcdf3.N3raf;
import ucar.nc2.write.Nc4Chunking;

import javax.annotation.Nonnull;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.util.*;

/**
 * Writes Netcdf 3 or 4 formatted files to disk.
 * To write new files:
 *

 *
createNew()

 *
Add objects with addXXX() deleteXXX() renameXXX() calls

 *
create file and write metadata with create()

 *
write data with writeXXX()

 *
close()

 *

 * To write data to existing files:
 *

 *
openExisting()

 *
write data with writeXXX()

 *
close()

 *

 *

 * NetcdfFileWriter is a low level wrap of IOServiceProviderWriter, if possible better to use:
 *

  *
ucar.nc2.FileWriter2()

  *
ucar.nc2.dt.grid.CFGridWriter

  *
ucar.nc2.ft.point.writer.CFPointWriter

  *
ucar.nc2.ft2.coverage.grid.CFGridCoverageWriter

  *

 * @author caron
 * @since 7/25/12
 */
public class NetcdfFileWriter implements Closeable {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(NetcdfFileWriter.class);
  static private Set<DataType> validN3types = EnumSet.of(DataType.BYTE, DataType.CHAR, DataType.SHORT, DataType.INT,
          DataType.DOUBLE, DataType.FLOAT);

  /**
   * The kinds of netcdf file that can be written.
   */
  public enum Version {
    netcdf3(".nc"),              // java iosp
    netcdf4(".nc4"),             // jni netcdf4 iosp mode = NC_FORMAT_NETCDF4
    netcdf4_classic(".nc"),      // jni netcdf4 iosp mode = NC_FORMAT_NETCDF4_CLASSIC
    netcdf3c(".nc"),             // jni netcdf4 iosp mode = NC_FORMAT_CLASSIC   (nc3)
    netcdf3c64(".nc"),           // jni netcdf4 iosp mode = NC_FORMAT_64BIT     (nc3 64 bit)
    ncstream(".ncs");            // ncstream iosp

    private String suffix;

    Version(String suffix) {
      this.suffix = suffix;
    }

    public boolean isNetdf4format() {
      return this == netcdf4 || this == netcdf4_classic;
    }

    public boolean isExtendedModel() {
      return this == netcdf4 || this == ncstream;
    }

    public boolean useJniIosp() {
      return this != netcdf3 && this != ncstream;
    }

    public String getSuffix() {
      return suffix;
    }
  }

  /**
   * Open an existing Netcdf file for writing data. Fill mode is true.
   * Cannot add new objects, you can only read/write data to existing Variables.
   *
   * @param location name of existing file to open.
   * @return existing file that can be written to
   * @throws java.io.IOException on I/O error
   */
  static public NetcdfFileWriter openExisting(String location) throws IOException {
    return new NetcdfFileWriter(null, location, true, null); // dont know the version yet
  }

  static public NetcdfFileWriter createNew(Version version, String location) throws IOException {
    return new NetcdfFileWriter(version, location, false, null);
  }

  static public NetcdfFileWriter createNew(String location, boolean fill) throws IOException {
    NetcdfFileWriter result = new NetcdfFileWriter(Version.netcdf3, location, false, null);
    result.setFill(fill);
    return result;
  }

  /**
   * Create a new Netcdf file, with fill mode true.
   *
   * @param version  netcdf-3 or 4
   * @param location name of new file to open; if it exists, will overwrite it.
   * @param chunker  used only for netcdf4, or null for default chunking algorithm
   * @return new NetcdfFileWriter
   * @throws IOException on I/O error
   */
  static public NetcdfFileWriter createNew(Version version, String location, Nc4Chunking chunker) throws IOException {
    return new NetcdfFileWriter(version, location, false, chunker);
  }

  ////////////////////////////////////////////////////////////////////////////////
  private final String location;
  private IOServiceProviderWriter spiw;

  // modes
  private boolean defineMode;

  // state
  private NetcdfFile ncfile;
  private Version version;
  private boolean isNewFile;
  private boolean isLargeFile;
  private boolean fill = true;
  private int extraHeader;
  private long preallocateSize;
  private Map<String,String> varRenameMap = new HashMap<>();

  /**
   * Open an existing or create a new Netcdf file
   *
   * @param version which kind of file to write, if null, use netcdf3 (isExisting= false) else open file and figure out the version
   * @param location   open a new file at this location
   * @param isExisting true if file already exists
   * @param chunker    used only for netcdf4, or null for used only for netcdf4, or null for default chunking algorithm
   * @throws IOException on I/O error
   */
  protected NetcdfFileWriter(Version version, String location, boolean isExisting, Nc4Chunking chunker) throws IOException {

    ucar.unidata.io.RandomAccessFile raf = null;
    if (isExisting) {
      raf = new ucar.unidata.io.RandomAccessFile(location, "rw");

      try {
        if (H5header.isValidFile(raf)) {
          if (version != null && !version.isNetdf4format())
            throw new IllegalArgumentException(location + " must be netcdf-4 file");
          else version = Version.netcdf4;

        } else if (N3header.isValidFile(raf)) {
          if (version != null && (version != Version.netcdf3))
            throw new IllegalArgumentException(location + " must be netcdf-3 file");
          else version = Version.netcdf3;

        } else {
          raf.close();
          throw new IllegalArgumentException(location + " must be netcdf-3 or netcdf-4 file");
        }

      } catch (IOException ioe) {
        raf.close();
        throw ioe;
      }

    } else {
      if (version == null) version = Version.netcdf3;
      isNewFile = true;
    }

    this.version = version;
    this.location = location;

    if (version.useJniIosp()) {
      IOServiceProviderWriter spi;
      try {
        //  Nc4Iosp.setLibraryAndPath(path, name);
        Class iospClass = this.getClass().getClassLoader().loadClass("ucar.nc2.jni.netcdf.Nc4Iosp");
        Constructor<IOServiceProviderWriter> ctor = iospClass.getConstructor(Version.class);
        spi = ctor.newInstance(version);

        Method method = iospClass.getMethod("setChunker", Nc4Chunking.class);
        method.invoke(spi, chunker);
      } catch (Throwable e) {
        throw new IllegalArgumentException("ucar.nc2.jni.netcdf.Nc4Iosp failed, cannot use version " + version, e);
      }
      spiw = spi;
    } else {
      spiw = new N3raf();
    }

    this.ncfile = new NetcdfFile(spiw, location);  // package private
    if (isExisting)
      spiw.openForWriting(raf, ncfile, null);
    else
      defineMode = true;
  }

  /**
   * Set the fill flag: call before calling create() or doing any data writing. Only used by netcdf-3 (?).
   * If true, the data is first written with fill values.
   * Default is fill = false.
   * Leave false if you expect to write all data values, set to true if you want to be
   * sure that unwritten data values have the fill value in it.
   *
   * @param fill set fill mode true or false
   */
  public void setFill(boolean fill) {
    this.fill = fill;
    spiw.setFill(fill);
  }

  /**
   * Preallocate the file size, for efficiency. Only used by netcdf-3.
   * Must be in define mode
   * Must call before create() to have any affect.
   *
   * @param size if set to > 0, set length of file to this upon creation - this (usually) pre-allocates contiguous storage.
   */
  public void setLength(long size) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    this.preallocateSize = size;
  }

  /**
   * Set if this should be a "large file" (64-bit offset) format. Only used by netcdf-3.
   * Must be in define mode
   *
   * @param isLargeFile true if large file
   */
  public void setLargeFile(boolean isLargeFile) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    this.isLargeFile = isLargeFile;
  }

  /**
   * Set extra bytes to reserve in the header. Only used by netcdf-3.
   * This can prevent rewriting the entire file on redefine.
   * Must be in define mode
   *
   * @param extraHeaderBytes # bytes extra for the header
   */
  public void setExtraHeaderBytes(int extraHeaderBytes) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    this.extraHeader = extraHeaderBytes;
  }

  /**
   * Is the file in define mode, which allows objects to be added and changed?
   *
   * @return true if the file in define mode
   */
  public boolean isDefineMode() {
    return defineMode;
  }

  public NetcdfFile getNetcdfFile() {
    if (defineMode) throw new IllegalStateException("Must leave define mode first");
    return ncfile;
  }

  public Version getVersion() {
    return version;
  }

  public Variable findVariable(String fullNameEscaped) {
    return ncfile.findVariable(fullNameEscaped);
  }

  public Dimension findDimension(String dimName) {
    return ncfile.findDimension(dimName);
  }

  public Attribute findGlobalAttribute(String attName) {
    return ncfile.getRootGroup().findAttribute(attName);
  }

  ////////////////////////////////////////////
  //// use these calls in define mode

  public Dimension addDimension(String dimName, int length) {
    return addDimension(null, dimName, length, false, false);
  }

  /**
   * Add a shared Dimension to the file. Must be in define mode.
   *
   * @param dimName name of dimension
   * @param length  size of dimension.
   * @return the created dimension
   */
  public Dimension addDimension(Group g, String dimName, int length) {
    return addDimension(g, dimName, length, false, false);
  }

  /**
   * Add single unlimited, shared dimension (classic model)
   * @param dimName name of dimension
   * @return Dimension object that was added
   */
  public Dimension addUnlimitedDimension(String dimName) {
    return addDimension(null, dimName, 0, true, false);
  }

  public Dimension addDimension(String dimName, int length, boolean isUnlimited, boolean isVariableLength) {
    return addDimension(null, dimName, length, isUnlimited, isVariableLength);
  }

  /**
   * Add a shared Dimension to the file. Must be in define mode.
   *
   * @param dimName          name of dimension
   * @param length           size of dimension.
   * @param isUnlimited      if dimension is unlimited
   * @param isVariableLength if dimension is variable length
   * @return the created dimension
   */
  public Dimension addDimension(Group g, String dimName, int length, boolean isUnlimited, boolean isVariableLength) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    if (!isValidObjectName(dimName))
      throw new IllegalArgumentException("illegal dimension name " + dimName);

    Dimension dim = new Dimension(dimName, length, true, isUnlimited, isVariableLength);
    ncfile.addDimension(g, dim);
    return dim;
  }

  public boolean hasDimension(Group g, String dimName) {
    if (g == null) g = ncfile.getRootGroup();
    return g.findDimension(dimName) != null;
  }

  private String makeValidObjectName(String name) {
    if (!isValidObjectName(name)) {
      String nname = createValidObjectName(name);
      log.warn("illegal object name= " + name + " change to " + name);
      return nname;
    }
    return name;
  }

  private boolean isValidObjectName(String name) {
    return N3iosp.isValidNetcdfObjectName(name);
  }

  private boolean isValidDataType(DataType dt) {
    return version.isExtendedModel() || validN3types.contains(dt);
  }

  private String createValidObjectName(String name) {
    return N3iosp.makeValidNetcdfObjectName(name);
  }

  /**
   * Rename a Dimension. Must be in define mode.
   *
   * @param oldName existing dimension has this name
   * @param newName rename to this
   * @return renamed dimension, or null if not found
   */
  public Dimension renameDimension(Group g, String oldName, String newName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    if (!isValidObjectName(newName)) throw new IllegalArgumentException("illegal dimension name " + newName);

    if (g == null) g = ncfile.getRootGroup();
    Dimension dim = g.findDimension(oldName);
    if (null != dim) dim.setName(newName);
    return dim;
  }

  /**
   * Add a Group to the file. Must be in define mode.
   * If pass in null as the parent then the root group is returned and the name is ignored.
   * This is how you get the root group. Note this is different from other uses of parent group.
   *
   * @param parent the parent of this group, if null then returns the root group.
   * @param name   the name of this group, unique within parent
   * @return the created group
   */
  public Group addGroup(Group parent, String name) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    if (parent == null) return ncfile.getRootGroup();

    Group result = new Group(ncfile, parent, name);
    parent.addGroup(result);
    return result;
  }

  public Attribute addGlobalAttribute(Attribute att) {
    return addGroupAttribute(null, att);
  }

  public Attribute addGlobalAttribute(String name, String value) {
    return addGroupAttribute(null, new Attribute(name, value));
  }

  public Attribute addGlobalAttribute(String name, Number value) {
    return addGroupAttribute(null, new Attribute(name, value));
  }

  /**
   * Add a Global attribute to the file. Must be in define mode.
   *
   * @param g   the group to add to. if null, use root group
   * @param att the attribute.
   * @return the created attribute
   */
  public Attribute addGroupAttribute(Group g, Attribute att) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");

    if (!isValidObjectName(att.getShortName())) {
      String attName = createValidObjectName(att.getShortName());
      log.warn("illegal attribute name= " + att.getShortName() + " change to " + attName);
      att = new Attribute(attName, att.getValues());
    }

    return ncfile.addAttribute(g, att);
  }

  /**
   * Add a EnumTypedef to the file. Must be in define mode.
   *
   * @param g  the group to add to. if null, use root group
   * @param td the EnumTypedef.
   * @return the created attribute
   */
  public EnumTypedef addTypedef(Group g, EnumTypedef td) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    if (!version.isExtendedModel())
      throw new IllegalArgumentException("Enum type only supported in extended model, this version is="+version);
    g.addEnumeration(td);
    return td;
  }

  public Attribute deleteGlobalAttribute(String attName) {
    return deleteGroupAttribute(null, attName);
  }

  /**
   * Delete a group Attribute. Must be in define mode.
   *
   * @param g       the group to add to. if null, use root group
   * @param attName existing Attribute has this name
   * @return deleted Attribute, or null if not found
   */
  public Attribute deleteGroupAttribute(Group g, String attName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    if (g == null) g = ncfile.getRootGroup();
    Attribute att = g.findAttribute(attName);
    if (null == att) return null;
    g.remove(att);
    return att;
  }

  public Attribute renameGlobalAttribute(String oldName, String newName) {
    return renameGroupAttribute(null, oldName, newName);
  }
  /**
   * Rename a group Attribute. Must be in define mode.
   *
   * @param g       the group to add to. if null, use root group
   * @param oldName existing Attribute has this name
   * @param newName rename to this
   * @return renamed Attribute, or null if not found
   */
  public Attribute renameGroupAttribute(Group g, String oldName, String newName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");

    if (!isValidObjectName(newName)) {
      String newnewName = createValidObjectName(newName);
      log.warn("illegal attribute name= " + newName + " change to " + newnewName);
      newName = newnewName;
    }

    if (g == null) g = ncfile.getRootGroup();
    Attribute att = g.findAttribute(oldName);
    if (null == att) return null;

    g.remove(att);
    att = new Attribute(newName, att.getValues());
    g.addAttribute(att);
    return att;
  }

  public Variable addVariable(String shortName, DataType dataType, String dimString) {
    return addVariable(null, shortName, dataType, dimString);
  }

  /**
   * Add a variable to the file. Must be in define mode.
   *
   * @param g         the group to add to. if null, use root group
   * @param shortName name of Variable, must be unique with the file.
   * @param dataType  type of underlying element
   * @param dimString names of Dimensions for the variable, blank separated.
   *                  Must already have been added. Use an empty string for a scalar variable.
   * @return the Variable that has been added
   */
  public Variable addVariable(Group g, String shortName, DataType dataType, String dimString) {
    Group parent = (g == null) ? ncfile.getRootGroup() : g;
    return addVariable(g, null, shortName, dataType, Dimension.makeDimensionsList(parent, dimString));
  }

  public Variable addVariable(String shortName, DataType dataType, List<Dimension> dims) {
    return addVariable(null, shortName, dataType, dims);
  }

  /**
   * Add a variable to the file. Must be in define mode.
   *
   * @param g         add to this group in the new file
   * @param shortName name of Variable, must be unique with the file.
   * @param dataType  type of underlying element
   * @param dims      list of Dimensions for the variable in the new file, must already have been added.
   *                  Use a list of length 0 for a scalar variable.
   * @return the Variable that has been added, or null if a Variable with shortName already exists in the group
   */
  public Variable addVariable(Group g, String shortName, DataType dataType, List<Dimension> dims) {
    if (g == null) g = ncfile.getRootGroup();
    Variable oldVar = g.findVariable(shortName);
    if (oldVar != null) return null;
    return addVariable(g, null, shortName, dataType, dims);
  }

  /**
   * Add a variable to the file. Must be in define mode.
   *
   * @param g         add to this group in the new file
   * @param parent    parent Structure (netcdf4 only), or null if not a member of a Structure
   * @param shortName name of Variable, must be unique with the file.
   * @param dataType  type of underlying element
   * @param dims      list of Dimensions for the variable in the new file, must already have been added.
   *                  Use a list of length 0 for a scalar variable.
   * @return the Variable that has been added
   */
  public Variable addVariable(Group g, Structure parent, String shortName, DataType dataType, List<Dimension> dims) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");

    DataType writeType =  version.isExtendedModel() ?
            dataType : dataType.withSignedness(DataType.Signedness.SIGNED); // use signed type for netcdf3
    boolean usingSignForUnsign = writeType != dataType;
    if (!isValidDataType(writeType))
      throw new IllegalArgumentException("illegal dataType: " + dataType + " not supported in netcdf-3");

    // check unlimited if classic model
    if (!version.isExtendedModel()) {
      for (int i = 0; i < dims.size(); i++) {
        Dimension d = dims.get(i);
        if (d.isUnlimited() && (i != 0))
          throw new IllegalArgumentException("Unlimited dimension " + d.getShortName() + " must be first (outermost) in netcdf-3 ");
      }
    }

    shortName = makeValidObjectName(shortName);
    Variable v;
    if (dataType == DataType.STRUCTURE) {
      v = new Structure(ncfile, g, parent, shortName);
    } else {
      v = new Variable(ncfile, g, parent, shortName);
    }
    v.setDataType(writeType);
    v.setDimensions(dims);
    if (usingSignForUnsign)
      v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));

    long size = v.getSize() * v.getElementSize();
    if (version == Version.netcdf3 && size > N3iosp.MAX_VARSIZE)
      throw new IllegalArgumentException("Variable size in bytes " + size + " may not exceed " + N3iosp.MAX_VARSIZE);
      //System.out.printf("Variable size in bytes " + size + " may not exceed " + N3iosp.MAX_VARSIZE);

    ncfile.addVariable(g, v);
    return v;
  }

  /**
   * Adds a copy of the specified structure to the file (netcdf4 only). DO NOT USE YET
   *
   * @param g         add to this group in the new file
   * @param original  the structure to make a copy of in the new file.
   * @param shortName name of Variable, must be unique with the file.
   * @param dims      list of Dimensions for the variable in the new file, must already have been added.
   *                  Use a list of length 0 for a scalar variable.
   * @return the Structure variable that has been added
   */
  public Structure addCopyOfStructure(Group g, @Nonnull Structure original, String shortName, List<Dimension> dims) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");
    if (original == null)
      throw new NullPointerException("Original structure must be non-null");

    shortName = makeValidObjectName(shortName);
    if (!version.isExtendedModel())
      throw new IllegalArgumentException("Structure type only supported in extended model, version="+version);

    Structure s = new Structure(ncfile, g, null, shortName);
    s.setDimensions(dims);

    for (Variable m : original.getVariables()) {  // LOOK no nested structs
      Variable nest = new Variable(ncfile, g, s, m.getShortName());
      nest.setDataType(m.getDataType());
      nest.setDimensions(m.getDimensions());
      nest.addAll(m.getAttributes());
      s.addMemberVariable(nest);
    }

    ncfile.addVariable(g, s);
    return s;
  }

  public Variable addStructureMember(Structure s, String shortName, DataType dtype, String dims) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");

    shortName = makeValidObjectName(shortName);
    if (!version.isExtendedModel())
      throw new IllegalArgumentException("Structure type only supported in extended model, version="+version);

    Variable m = new Variable(ncfile, null, s, shortName, dtype, dims);  // LOOK: What if dtype == STRUCTURE?
    s.addMemberVariable(m);

    // We've added a member to s. Recalculate its size and all ancestor structure sizes.
    for (Structure struct = s; struct != null; struct = struct.getParentStructure()) {
      struct.calcElementSize();
    }

    return m;
  }

  /**
   * Add a variable with DataType = String to a netCDF-3 file. Must be in define mode.
   * The variable will be stored in the file as a CHAR variable.
   * A new dimension with name "stringVar.getShortName()_strlen" is automatically
   * added, with length max_strlen, as determined from the data contained in the
   * stringVar.
   *
   * @param g         add to this group in the new file
   * @param stringVar string variable.
   * @param dims      list of Dimensions for the string variable.
   * @return the CHAR variable generated from stringVar
   */
  public Variable addStringVariable(Group g, Variable stringVar, List<Dimension> dims) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");

    if (!N3iosp.isValidNetcdfObjectName(stringVar.getShortName()))
      throw new IllegalArgumentException("illegal netCDF-3 variable name: " + stringVar.getShortName());

    // convert STRING to CHAR
    int max_strlen = 0;
    Array data;

    try {
      data = stringVar.read();
      IndexIterator ii = data.getIndexIterator();
      while (ii.hasNext()) {
        String s = (String) ii.getObjectNext();
        max_strlen = Math.max(max_strlen, s.length());
      }
    } catch (IOException e) {
      e.printStackTrace();
      String err = "No data found for Variable " + stringVar.getShortName() +
              ". Cannot determine the lentgh of the new CHAR variable.";
      log.error(err);
      System.out.println(err);
    }

    return addStringVariable(g, stringVar.getShortName(), dims, max_strlen);
  }

  /**
   * Add a variable with DataType = String to the file. Must be in define mode.
   * The variable will be stored in the file as a CHAR variable.
   * A new dimension with name "varName_strlen" is automatically added, with length max_strlen.
   *
   * @param shortName  name of Variable, must be unique within the file.
   * @param dims       list of Dimensions for the variable, must already have been added. Use a list of length 0
   *                   for a scalar variable. Do not include the string length dimension.
   * @param max_strlen maximum string length.
   * @return the Variable that has been added
   */
  public Variable addStringVariable(Group g, String shortName, List<Dimension> dims, int max_strlen) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");
    shortName = makeValidObjectName(shortName);

    Variable v = new Variable(ncfile, g, null, shortName);
    v.setDataType(DataType.CHAR);

    Dimension d = addDimension(g, shortName + "_strlen", max_strlen);
    List<Dimension> sdims = new ArrayList<>(dims);
    sdims.add(d);
    v.setDimensions(sdims);

    ncfile.addVariable(g, v);
    return v;
  }

  /* public List<Dimension> makeDimList(Group g, String dimNames) {
    if (g == null) g = ncfile.getRootGroup();
    List<Dimension> list = new ArrayList<>();
    StringTokenizer stoker = new StringTokenizer(dimNames);
    while (stoker.hasMoreTokens()) {
      String tok = stoker.nextToken();
      Dimension d = g.findDimension(tok);
      if (null == d) {
        g.findDimension(tok); // debug
        throw new IllegalArgumentException("Cant find dimension " + tok);
      }
      list.add(d);
    }
    return list;
  } */

  /**
   * Rename a Variable. Must be in define mode.
   *
   * @param oldName existing Variable has this name
   * @param newName rename to this
   * @return renamed Variable, or null if not found
   */
  public Variable renameVariable(String oldName, String newName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");
    Variable v = ncfile.findVariable(oldName);
    if (null != v) {
      String fullOldNameEscaped = v.getFullNameEscaped();
      v.setName(newName);
      varRenameMap.put(v.getFullNameEscaped(), fullOldNameEscaped);
    }
    return v;
  }

  public boolean addVariableAttribute(String varName, String name, String value) {
    return addVariableAttribute(findVariable(varName), new Attribute(name, value));
  }

  public boolean addVariableAttribute(String varName, String name, Number value) {
    return addVariableAttribute(findVariable(varName), new Attribute(name, value));
  }

  public boolean addVariableAttribute(String varName, Attribute att) {
    return addVariableAttribute(findVariable(varName), att);
  }
  /**
   * Add an attribute to the named Variable. Must be in define mode.
   *
   * @param v   Variable to add attribute to
   * @param att Attribute to add.
   * @return true if attribute was added, false if not allowed by CDM.
   */
  public boolean addVariableAttribute(Variable v, Attribute att) {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");

    if (!isValidObjectName(att.getShortName())) {
      String attName = createValidObjectName(att.getShortName());
      log.warn("illegal netCDF-3 attribute name= " + att.getShortName() + " change to " + attName);
      att = new Attribute(attName, att.getValues());
    }

    v.addAttribute(att);
    return true;
  }

  /**
   * Delete a variable Attribute. Must be in define mode.
   *
   * @param v       Variable to delete attribute to
   * @param attName existing Attribute has this name
   * @return deleted Attribute, or null if not found
   */
  public Attribute deleteVariableAttribute(Variable v, String attName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");

    Attribute att = v.findAttribute(attName);
    if (null == att) return null;

    v.remove(att);
    return att;
  }

  public Variable deleteVariable(String fullName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");

    Variable v = ncfile.findVariable(fullName);
    if (v != null)
      ncfile.removeVariable(v.getParentGroup(), v.getShortName());
    return v;
  }

  /**
   * Rename a variable Attribute. Must be in define mode.
   *
   * @param v       Variable to modify attribute
   * @param attName existing Attribute has this name
   * @param newName rename to this
   * @return renamed Attribute, or null if not found
   */
  public Attribute renameVariableAttribute(Variable v, String attName, String newName) {
    if (!defineMode) throw new UnsupportedOperationException("not in define mode");

    Attribute att = v.findAttribute(attName);
    if (null == att) return null;

    v.remove(att);
    att = new Attribute(newName, att.getValues());
    v.addAttribute(att);
    return att;
  }

  /**
   * Update the value of an existing attribute. Attribute is found by name, which must match exactly.
   * You cannot make an attribute longer, or change the number of values.
   * For strings: truncate if longer, zero fill if shorter.  Strings are padded to 4 byte boundaries, ok to use padding if it exists.
   * For numerics: must have same number of values.
   * This is really a netcdf-3 writing only. netcdf-4 attributes can be changed without rewriting.
   *
   * @param v2  variable, or null for global attribute
   * @param att replace with this value
   * @throws IOException if I/O error
   */
  public void updateAttribute(ucar.nc2.Variable v2, Attribute att) throws IOException {
  //  if (defineMode)
  //    throw new UnsupportedOperationException("in define mode");
    spiw.updateAttribute(v2, att);
  }

  /**
   * After you have added all of the Dimensions, Variables, and Attributes,
   * call create() to actually create the file. You must be in define mode.
   * After this call, you are no longer in define mode.
   *
   * @throws java.io.IOException if I/O error
   */
  public void create() throws java.io.IOException {
    if (!defineMode)
      throw new UnsupportedOperationException("not in define mode");
    if (!isNewFile)
      throw new UnsupportedOperationException("can only call create on a new file");

    ncfile.finish(); // ??
    spiw.setFill(fill); // ??
    spiw.create(location, ncfile, extraHeader, preallocateSize, isLargeFile);

    defineMode = false;
  }

  ////////////////////////////////////////////
  // redefine

  /**
   * Set the redefine mode.
   * Designed to emulate nc_redef (redefineMode = true) and
   * nc_enddef (redefineMode = false)
   *
   * @param redefineMode start or end define mode
   * @return true if it had to rewrite the entire file, false if it wrote the header in place
   * @throws java.io.IOException on read/write error
   */
  public boolean setRedefineMode(boolean redefineMode) throws IOException {
    if (redefineMode && !defineMode) {
      defineMode = true;

    } else if (!redefineMode && defineMode) {
      defineMode = false;
      ncfile.finish();

      // try to rewrite header, if it fails, then we have to rewrite entire file
      boolean ok = spiw.rewriteHeader(isLargeFile);  // LOOK seems like we should be using isNewFile
      if (!ok)
        rewrite();
      return !ok;
    }

    return false;
  }

  // rewrite entire file
  private void rewrite() throws IOException {
    // close existing file, rename and open as read-only
    spiw.flush();
    spiw.close();

    File prevFile = new File(location);
    if (!prevFile.exists()) {
      return;
    }

    File tmpFile = new File(location + ".tmp");
    if (tmpFile.exists()) {
      boolean ok = tmpFile.delete();
      if (!ok) log.warn("rewrite unable to delete {}", tmpFile.getPath());
    }
    if (!prevFile.renameTo(tmpFile)) {
      System.out.println(prevFile.getPath() + " prevFile.exists " + prevFile.exists() + " canRead = " + prevFile.canRead());
      System.out.println(tmpFile.getPath() + " tmpFile.exists " + tmpFile.exists() + " canWrite " + tmpFile.canWrite());
      throw new RuntimeException("Cant rename " + prevFile.getAbsolutePath() + " to " + tmpFile.getAbsolutePath());
    }

    NetcdfFile oldFile = NetcdfFile.open(tmpFile.getPath());

    /* use record dimension if it has one
    Structure recordVar = null;
    if (oldFile.hasUnlimitedDimension()) {
      oldFile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      recordVar = (Structure) oldFile.findVariable("record");
      /* if (recordVar != null) {
        Boolean result = (Boolean) spiw.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
        if (!result)
          recordVar = null;
      }
      } */

    // create new file with current set of objects
    spiw.create(location, ncfile, extraHeader, preallocateSize, isLargeFile);
    spiw.setFill(fill);
    //isClosed = false;

    /* wait till header is written before adding the record variable to the file
    if (recordVar != null) {
      Boolean result = (Boolean) spiw.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      if (!result)
        recordVar = null;
    } */

    FileWriter2 fileWriter2 = new FileWriter2(this);

    for (Variable v : ncfile.getVariables()) {
      String oldVarName = v.getFullName();
      Variable oldVar = oldFile.findVariable(oldVarName);
      if (oldVar != null) {
        fileWriter2.copyAll(oldVar, v);
      } else if (varRenameMap.containsKey(oldVarName)) {
        // var name has changed in ncfile - use the varRenameMap to find
        //  the correct variable name to request from oldFile
        String realOldVarName = varRenameMap.get(oldVarName);
        oldVar = oldFile.findVariable(realOldVarName);
        if (oldVar != null) {
          fileWriter2.copyAll(oldVar, v);
        }
      } else {
        String message = "Cannot find variable " + oldVarName + " to copy to new file.";
        log.warn(message);
        System.out.println(message);
      }
    }

    // delete old
    oldFile.close();
    if (!tmpFile.delete())
      throw new RuntimeException("Cant delete "+tmpFile.getAbsolutePath());
  }

  /**
   * For netcdf3 only, take all unlimited variables and make them into a structure.
   * @return the record Structure, or null if not done.
   */
  public Structure addRecordStructure() {
    if (version != Version.netcdf3) return null;
    boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
    if (!ok)
      throw new IllegalStateException("can't add record variable");
    return (Structure) ncfile.findVariable("record");
  }

  ////////////////////////////////////////////
  //// use these calls to write data to the file

  public void write(String varname, Array values) throws java.io.IOException, InvalidRangeException {
    write(findVariable(varname), values);
  }
  /**
   * Write data to the named variable, origin assumed to be 0. Must not be in define mode.
   *
   * @param v      variable to write to
   * @param values write this array; must be same type and rank as Variable
   * @throws IOException                    if I/O error
   * @throws ucar.ma2.InvalidRangeException if values Array has illegal shape
   */
  public void write(Variable v, Array values) throws java.io.IOException, InvalidRangeException {
    if (ncfile != v.getNetcdfFile())
      throw new IllegalArgumentException("Variable is not owned by this writer.");

    write(v, new int[values.getRank()], values);
  }

  public void write(String varName, int[] origin, Array values) throws java.io.IOException, InvalidRangeException {
    write(findVariable(varName), origin, values);
  }

  /**
   * Write data to the named variable. Must not be in define mode.
   *
   * @param v      variable to write to
   * @param origin offset within the variable to start writing.
   * @param values write this array; must be same type and rank as Variable
   * @throws IOException                    if I/O error
   * @throws ucar.ma2.InvalidRangeException if values Array has illegal shape
   */
  public void write(Variable v, int[] origin, Array values) throws java.io.IOException, InvalidRangeException {
    if (defineMode)
      throw new UnsupportedOperationException("in define mode");

    spiw.writeData(v, new Section(origin, values.getShape()), values);
    v.invalidateCache();
  }

  /**
   * Write String data to a CHAR variable, origin assumed to be 0. Must not be in define mode.
   *
   * @param v      variable to write to
   * @param values write this array; must be ArrayObject of String
   * @throws IOException                    if I/O error
   * @throws ucar.ma2.InvalidRangeException if values Array has illegal shape
   */
  public void writeStringData(Variable v, Array values) throws java.io.IOException, InvalidRangeException {
    writeStringData(v, new int[values.getRank()], values);
  }

  /**
   * Write String data to a CHAR variable. Must not be in define mode.
   *
   * @param v      variable to write to
   * @param origin offset to start writing, ignore the strlen dimension.
   * @param values write this array; must be ArrayObject of String
   * @throws IOException                    if I/O error
   * @throws ucar.ma2.InvalidRangeException if values Array has illegal shape
   */
  public void writeStringData(Variable v, int[] origin, Array values) throws java.io.IOException, InvalidRangeException {

    if (values.getElementType() != String.class)
      throw new IllegalArgumentException("Must be ArrayObject of String ");

    if (v.getDataType() != DataType.CHAR)
      throw new IllegalArgumentException("variable " + v.getFullName() + " is not type CHAR");
    int rank = v.getRank();
    int strlen = v.getShape(rank - 1);

    // turn it into an ArrayChar
    ArrayChar cvalues = ArrayChar.makeFromStringArray((ArrayObject) values, strlen);

    int[] corigin = new int[rank];
    System.arraycopy(origin, 0, corigin, 0, rank - 1);

    write(v, corigin, cvalues);
  }

  public int appendStructureData(Structure s, StructureData sdata) throws IOException, InvalidRangeException {
    return spiw.appendStructureData(s, sdata);
  }

  /**
   * Flush anything written to disk.
   *
   * @throws IOException if I/O error
   */
  public void flush() throws java.io.IOException {
    spiw.flush();
  }

  /**
   * close the file.
   *
   * @throws IOException if I/O error
   */
  public synchronized void close() throws java.io.IOException {
    if (spiw != null) {
      setRedefineMode(false);
      flush();
      spiw.close();
      spiw = null;
    }
  }

  /**
   * Abort writing to this file. The file is closed.
   * @throws java.io.IOException
   */
  public void abort() throws java.io.IOException {
    if (spiw != null) {
      spiw.close();
      spiw = null;
    }
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.util.prefs;

import java.beans.BeanInfo;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;

class Bean {

  private Object o; // the wrapped object
  private BeanParser p = null; // the bean parser (shared for all beans of same class)

  // wrap an object in a Bean
  public Bean(Object o) {
    this.o = o;
  }

  // create a bean from an XML element
  public Bean(org.xml.sax.Attributes atts) throws ClassNotFoundException, InstantiationException, IllegalAccessException {
    String className = atts.getValue("class");
    Class<?> c = Class.forName(className);
    o = c.newInstance();
    p = BeanParser.getParser( c);
    p.readProperties(o, atts);
  }

  // write XML using the bean properties of the contained object
  public void writeProperties(PrintWriter out) throws IOException {
    if (p == null) p = BeanParser.getParser( o.getClass());
    p.writeProperties(o, out);
  }

  // get the wrapped object
  public Object getObject() { return o; }
  public Class<?> getBeanClass() { return o.getClass(); }

  static class Collection {
    private java.util.Collection<Object> collect; // the underlying collection
    private Class<?> beanClass; // the class of the beans in the collection
    private BeanParser p = null; // the bean parser (shared for all beans of same class)

    // wrap a collection in a bean
    Collection(java.util.Collection<Object> collect) {
      this.collect = collect;
      if (!collect.isEmpty()) {
        Iterator iter = collect.iterator();
        beanClass = iter.next().getClass();
        p = BeanParser.getParser(beanClass);
      }
    }

    // create a bean collection from an XML element
    Collection(org.xml.sax.Attributes atts) throws ClassNotFoundException { // , InstantiationException, IllegalAccessException {
      String className = atts.getValue("class");
      beanClass = Class.forName( className);
      p = BeanParser.getParser( beanClass);
      collect = new ArrayList<>();
    }

    // write XML using the bean properties of the specified object
    public void writeProperties(PrintWriter out, Object thiso) throws IOException {
      p.writeProperties(thiso, out);
    }

    // get the underlying java.util.Collection
    public java.util.Collection getCollection() { return collect; }
    public Class<?> getBeanClass() { return beanClass; }

      // write XML using the bean properties of the contained object
    public Object readProperties(org.xml.sax.Attributes atts) throws InstantiationException, IllegalAccessException {
      Object o = beanClass.newInstance();
      p.readProperties(o, atts);
      collect.add(o);
      return o;
    }

  }

  private static class BeanParser {
    private static boolean debugBean = false;
    private static Map<Class<?>, BeanParser> parsers = new HashMap<>();

    static BeanParser getParser(Class<?> beanClass) {
      BeanParser parser;
      if (null == (parser = parsers.get(beanClass))) {
        parser = new BeanParser(beanClass);
        parsers.put(beanClass, parser);
      }
      return parser;
    }

    private Map<String, PropertyDescriptor> properties = new TreeMap<>();
    private Object[] args = new Object[1];
    BeanParser(Class<?> beanClass) {

      // get bean info
      BeanInfo info;
      try {
        info = Introspector.getBeanInfo(beanClass, Object.class);
      } catch (IntrospectionException e) {
        e.printStackTrace();
        return;
      }

      if (debugBean)
        System.out.println( "Bean "+beanClass.getName());

      // properties must have read and write method
      PropertyDescriptor[] pds = info.getPropertyDescriptors();
      for (PropertyDescriptor pd : pds) {
        if ((pd.getReadMethod() != null) && (pd.getWriteMethod() != null)) {
          properties.put(pd.getName(), pd);
          if (debugBean) {
            System.out.println(" property " + pd.getName());
          }
        }
      }
    }

    void writeProperties(Object bean, PrintWriter out) throws IOException {
      for (PropertyDescriptor pds : properties.values()) {
        Method getter = pds.getReadMethod();
        try {
          Object value = getter.invoke(bean, (Object[]) null);
          if (value == null) {
            continue;
          }
          if (value instanceof String) {
            value = XMLStore.quote((String) value);
          } else if (value instanceof Date) {
            value = ((Date) value).getTime();
          }
          out.print(pds.getName() + "='" + value + "' ");
          if (debugBean) {
            System.out.println(" property get " + pds.getName() + "='" + value + "' ");
          }
        } catch (InvocationTargetException | IllegalAccessException e) {
          e.printStackTrace();
        }
      }
    }

    void readProperties(Object bean, org.xml.sax.Attributes atts) {
      for (Object o1 : properties.values()) {
        PropertyDescriptor pds = (PropertyDescriptor) o1;
        Method setter = pds.getWriteMethod();
        if (setter == null) {
          continue;
        }

        try {
          String sArg = atts.getValue(pds.getName());
          Object arg = getArgument(pds.getPropertyType(), sArg);
          if (debugBean) {
            System.out.println(" property set " + pds.getName() + "=" + sArg + " == " + arg);
          }
          if (arg == null) {
            return;
          }
          args[0] = arg;
          setter.invoke(bean, args);
        } catch (NumberFormatException | InvocationTargetException | IllegalAccessException e) {
          e.printStackTrace();
        }
      }
    }

    private Object getArgument(Class<?> c, String value) {
      if (c == String.class) {
        return value;
      } else if (c == int.class) {
        return Integer.valueOf(value);
      } else if (c == double.class) {
        return Double.valueOf(value);
      } else if (c == boolean.class) {
        if (value == null) {
          // Boolean.valueOf(null) actually returns "false", which we don't want.
          return null;
        } else {
          return Boolean.valueOf(value);
        }
      } else if (c == float.class) {
        return Float.valueOf(value);
      } else if (c == short.class) {
        return Short.valueOf(value);
      } else if (c == long.class) {
        return Long.valueOf(value);
      } else if (c == byte.class) {
        return Byte.valueOf(value);
      } else if (c == Date.class) {
        long time = Long.parseLong(value);
        return new Date(time);
      } else {
        return null;
      }
    }
  }
}


/*
 * Copyright (c) 1998 - 2014. University Corporation for Atmospheric Research/Unidata
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package thredds.server.catalog.writer;

import thredds.client.catalog.Dataset;
import thredds.client.catalog.ThreddsMetadata;
import thredds.client.catalog.tools.DataFactory;
import ucar.nc2.Attribute;
import ucar.nc2.VariableSimpleIF;
import ucar.nc2.constants.DataFormatType;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dataset.CoordinateAxis1D;
import ucar.nc2.dataset.CoordinateAxis1DTime;
import ucar.nc2.dt.*;
import ucar.nc2.ft.FeatureDatasetPoint;
import ucar.nc2.ft.point.PointDatasetImpl;
import ucar.nc2.grib.collection.Grib;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.time.CalendarDateUnit;
import ucar.unidata.geoloc.LatLonRect;

import java.io.IOException;
import java.util.*;

/**
 * Extract Thredds Metadata from a feature dataset
 *
 * @author John
 * @since 1/19/2015
 */
public class ThreddsMetadataExtractor {
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ThreddsMetadataExtractor.class);

  /**
   * extract info from underlying feature dataset
   * @param threddsDataset  call DataFactory().openFeatureDataset() to open it
   * @return results in ThreddsMetadata object
   * @throws IOException
   */
  public ThreddsMetadata extract(Dataset threddsDataset) throws IOException {
    ThreddsMetadata metadata = new ThreddsMetadata();
    Map<String, Object> flds = metadata.getFlds();

    try ( DataFactory.Result result = new DataFactory().openFeatureDataset(threddsDataset, null)) {
      if (result.fatalError) {
        logger.warn(" openFeatureDataset failed, errs=%s%n", result.errLog);
        return null;
      }

      if (result.featureType.isCoverageFeatureType()) {
        GridDataset gridDataset = (GridDataset) result.featureDataset; // LOOK wrong
        flds.put(Dataset.GeospatialCoverage, extractGeospatial(gridDataset));

        CalendarDateRange tc = extractCalendarDateRange(gridDataset);
        if (tc != null)
          flds.put(Dataset.TimeCoverage, tc);

        ThreddsMetadata.VariableGroup vars = extractVariables(threddsDataset.getDataFormatName(), gridDataset);
        if (vars != null)
          flds.put(Dataset.VariableGroups, vars);

      } else if (result.featureType.isPointFeatureType()) {
        PointDatasetImpl pobsDataset = (PointDatasetImpl) result.featureDataset;
        LatLonRect llbb = pobsDataset.getBoundingBox();
        if (null != llbb)
          flds.put(Dataset.GeospatialCoverage, new ThreddsMetadata.GeospatialCoverage(llbb, null, 0.0, 0.0));

        CalendarDateRange tc = extractCalendarDateRange(pobsDataset);
        if (tc != null)
          flds.put(Dataset.TimeCoverage, tc);

        ThreddsMetadata.VariableGroup vars = extractVariables(pobsDataset);
        if (vars != null)
          flds.put(Dataset.VariableGroups, vars);
      }

    } catch (IOException ioe) {
      logger.error("Error opening dataset " + threddsDataset.getName(), ioe);
    }

    return metadata;
  }

  public ThreddsMetadata.GeospatialCoverage extractGeospatial(GridDataset gridDataset) {
    LatLonRect llbb = null;
    CoordinateAxis1D vaxis = null;

    for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
      GridCoordSystem gsys = gridset.getGeoCoordSystem();
      if (llbb == null)
        llbb = gsys.getLatLonBoundingBox();

      CoordinateAxis1D vaxis2 = gsys.getVerticalAxis();
      if (vaxis == null)
        vaxis = vaxis2;
      else if ((vaxis2 != null) && (vaxis2.getSize() > vaxis.getSize()))
        vaxis = vaxis2;
    }

    return new ThreddsMetadata.GeospatialCoverage(llbb, vaxis, 0.0, 0.0); // LOOK can we extract dx, dy ?
  }

  public ThreddsMetadata.VariableGroup extractVariables(String fileFormat, GridDataset gridDataset) {
    List<ThreddsMetadata.Variable> vars = new ArrayList<>();
    String vocab = fileFormat;
    DataFormatType fileType = DataFormatType.getType(fileFormat);

    if ((fileType != null) && ((fileType == DataFormatType.GRIB1) || fileType == DataFormatType.GRIB2)) {
      for (GridDatatype grid : gridDataset.getGrids()) {
        String name = grid.getShortName();
        String desc = grid.getDescription();
        String units = grid.getUnitsString();
        String vname = null;
        String id = null;

        Attribute att = grid.findAttributeIgnoreCase(Grib.VARIABLE_ID_ATTNAME);
        if (att != null) {
          id = att.getStringValue();
          vname = att.getStringValue();
        }
        vars.add(new ThreddsMetadata.Variable(name, desc, vname, units, id));
      }

      // String vocabRef = "http://www.wmo.int/pages/prog/www/WMOCodes/WMO306_vI2/LatestVERSION/LatestVERSION.html";

    } else { // GRID but not GRIB
      vocab = "CF-1.0";
      for (GridDatatype grid : gridDataset.getGrids()) {
        String name = grid.getShortName();
        String desc = grid.getDescription();
        String units = grid.getUnitsString();
        String vname = null;
        String id = null;

        ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name");
        if (att != null)
          vname = att.getStringValue();
        vars.add(new ThreddsMetadata.Variable(name, desc, vname, units, id));
      }
    }

    Collections.sort(vars);
                                             // String vocab, String vocabHref, URI vocabUri, URI mapUri, List<Variable> variables
    return new ThreddsMetadata.VariableGroup(vocab, null, null, vars);
  }

  ///////////////////////////////////////////////////////////////////////////////

  public ThreddsMetadata.GeospatialCoverage extractGeospatial(FeatureDatasetPoint fd) {
    LatLonRect llbb = fd.getBoundingBox();
    if (llbb != null) {
      return new ThreddsMetadata.GeospatialCoverage(llbb, null, 0.0, 0.0);
    }
    return null;
  }

  public ThreddsMetadata.VariableGroup extractVariables(FeatureDatasetPoint fd) {
    List<ThreddsMetadata.Variable> vars = new ArrayList<>();

    List<VariableSimpleIF> dataVars = fd.getDataVariables();
    if (dataVars == null)
      return null;

    for (VariableSimpleIF v : dataVars) {
      String name = v.getShortName();
      String desc = v.getDescription();
      String units = v.getUnitsString();
      String vname = null;
      String id = null;

      ucar.nc2.Attribute att = v.findAttributeIgnoreCase("standard_name");
      if (att != null)
        vname = att.getStringValue();
      vars.add(new ThreddsMetadata.Variable(name, desc, vname, units, id));
    }

    Collections.sort(vars);
                                             // String vocab, String vocabHref, URI vocabUri, URI mapUri, List<Variable> variables
    return new ThreddsMetadata.VariableGroup("CF-1.0", null, null, vars);
  }

  /* public DateRange extractDateRange(GridDataset gridDataset) {
    DateRange maxDateRange = null;

    for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
      GridCoordSystem gsys = gridset.getGeoCoordSystem();
      DateRange dateRange;

      CoordinateAxis1DTime time1D = gsys.getTimeAxis1D();
      if (time1D != null) {
        dateRange = time1D.getDateRange();
      } else {
        CoordinateAxis time = gsys.getTimeAxis();
        if (time == null)
          continue;

        try {
          DateUnit du = new DateUnit(time.getUnitsString());
          Date minDate = du.makeDate(time.getMinValue());
          Date maxDate = du.makeDate(time.getMaxValue());
          dateRange = new DateRange(minDate, maxDate);
        } catch (Exception e) {
          logger.warn("Illegal Date Unit " + time.getUnitsString());
          continue;
        }
      }

      if (maxDateRange == null)
        maxDateRange = dateRange;
      else
        maxDateRange.extend(dateRange);
    }

    return maxDateRange;
  } */

  public CalendarDateRange extractCalendarDateRange(FeatureDatasetPoint fd) {
    return fd.getCalendarDateRange();
  }

  ////////////////////

  public CalendarDateRange extractCalendarDateRange(GridDataset gridDataset) {
    CalendarDateRange maxDateRange = null;

    for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
      GridCoordSystem gsys = gridset.getGeoCoordSystem();
      CalendarDateRange dateRange;

      CoordinateAxis1DTime time1D = gsys.getTimeAxis1D();
      if (time1D != null) {
        dateRange = time1D.getCalendarDateRange();
      } else {
        CoordinateAxis time = gsys.getTimeAxis();
        if (time == null)
          continue;

        try {
          CalendarDateUnit du = CalendarDateUnit.of(null, time.getUnitsString()); // LOOK no calendar
          CalendarDate minDate = du.makeCalendarDate(time.getMinValue());
          CalendarDate maxDate = du.makeCalendarDate(time.getMaxValue());
          dateRange = CalendarDateRange.of(minDate, maxDate);
        } catch (Exception e) {
          logger.warn("Illegal Date Unit " + time.getUnitsString());
          continue;
        }
      }

      if (maxDateRange == null)
        maxDateRange = dateRange;
      else
        maxDateRange = maxDateRange.extend(dateRange);
    }

    return maxDateRange;
  }
}