/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.noaa;

import com.google.protobuf.InvalidProtocolBufferException;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.ncml.NcmlConstructor;
import ucar.nc2.stream.NcStream;
import ucar.nc2.util.CancelTask;
import ucar.unidata.io.RandomAccessFile;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * NOMADS Ghcnm2
 * LOOK probable file leaks
 * @author caron
 * @since Feb 26, 2011
 */

  /*
  ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/v3/README

    2.1 METADATA

       The metadata has been carried over from GHCN-Monthly v2.  This would
       include basic geographical station information such as latitude,
       longitude, elevation, station name, etc., and also extended metadata
       information, such as surrounding vegetation, etc.

    2.1.1 METADATA FORMAT

       Variable          Columns      Type
       --------          -------      ----

       ID                 1-11        Integer
       LATITUDE          13-20        Real
       LONGITUDE         22-30        Real
       STNELEV           32-37        Real
       NAME              39-68        Character
       GRELEV            70-73        Integer
       POPCLS            74-74        Character
       POPSIZ            76-79        Integer
       TOPO              80-81        Character
       STVEG             82-83        Character
       STLOC             84-85        Character
       OCNDIS            86-87        Integer
       AIRSTN            88-88        Character
       TOWNDIS           89-90        Integer
       GRVEG             91-106       Character
       POPCSS            107-107      Character

       Variable Definitions:

       ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
           the WMO stnId if the station is a WMO station.  It is a WMO station if
           digits 9-11="000".

       LATITUDE: latitude of station in decimal degrees

       LONGITUDE: longitude of station in decimal degrees

       STELEV: is the station elevation in meters. -999.0 = missing.

       NAME: station name

       GRELEV: station elevation in meters estimated from gridded digital
               terrain data

       POPCLS: population class
               (U=Urban (>50,000 persons);
               (S=Suburban (>=10,000 and <= 50,000 persons);
               (R=Rural (<10,000 persons)
               City and town boundaries are determined from location of station
               on Operational Navigation Charts with a scale of 1 to 1,000,000.
               For cities > 100,000 persons, population data were provided by
               the United Nations Demographic Yearbook. For smaller cities and
               towns several atlases were uses to determine population.

       POPSIZ: the population of the city or town the station is location in
               (expressed in thousands of persons).

       TOPO: type of topography in the environment surrounding the station,
             (Flat-FL,Hilly-HI,Mountain Top-MT,Mountainous Valley-MV).

       STVEG: type of vegetation in environment of station if station is Rural
              and when it is indicated on the Operational Navigation Chart
              (Desert-DE,Forested-FO,Ice-IC,Marsh-MA).

       STLOC: indicates whether station is near lake or ocean (<= 30 km of
              ocean-CO, adjacent to a lake at least 25 square km-LA).

       OCNDIS: distance to nearest ocean/lake from station (km).

       AIRSTN: airport station indicator (A=station at an airport).

       TOWNDIS: distance from airport to center of associated city or town (km).

       GRVEG: vegetation type at nearest 0.5 deg x 0.5 deg gridded data point of
              vegetation dataset (44 total classifications).

              BOGS, BOG WOODS
              COASTAL EDGES
              COLD IRRIGATED
              COOL CONIFER
              COOL CROPS
              COOL DESERT
              COOL FIELD/WOODS
              COOL FOR./FIELD
              COOL GRASS/SHRUB
              COOL IRRIGATED
              COOL MIXED
              EQ. EVERGREEN
              E. SOUTH. TAIGA
              HEATHS, MOORS
              HIGHLAND SHRUB
              HOT DESERT
              ICE
              LOW SCRUB
              MAIN TAIGA
              MARSH, SWAMP
              MED. GRAZING
              NORTH. TAIGA
              PADDYLANDS
              POLAR DESERT
              SAND DESERT
              SEMIARID WOODS
              SIBERIAN PARKS
              SOUTH. TAIGA
              SUCCULENT THORNS
              TROPICAL DRY FOR
              TROP. MONTANE
              TROP. SAVANNA
              TROP. SEASONAL
              TUNDRA
              WARM CONIFER
              WARM CROPS
              WARM DECIDUOUS
              WARM FIELD WOODS
              WARM FOR./FIELD
              WARM GRASS/SHRUB
              WARM IRRIGATED
              WARM MIXED
              WATER
              WOODED TUNDRA

       POPCSS: population class as determined by Satellite night lights
               (C=Urban, B=Suburban, A=Rural)

    2.2  DATA

         The data within GHCNM v3 beta for the time being consist of monthly
         average temperature, for the 7280 stations contained within GHCNM v2.
         Several new sources have been added to v3 beta, and a new "3 flag"
         format has been introduced, similar to that used within the Global
         Historical Climatology Network-Daily (GHCND).

    2.2.1 DATA FORMAT

          Variable          Columns      Type
          --------          -------      ----

          ID                 1-11        Integer
          YEAR              12-15        Integer
          ELEMENT           16-19        Character
          VALUE1            20-24        Integer
          DMFLAG1           25-25        Character
          QCFLAG1           26-26        Character
          DSFLAG1           27-27        Character
            .                 .             .
            .                 .             .
            .                 .             .
          VALUE12          108-112       Integer
          DMFLAG12         113-113       Character
          QCFLAG12         114-114       Character
          DSFLAG12         115-115       Character

          Variable Definitions:

          ID: 11 digit identifier, digits 1-3=Country Code, digits 4-8 represent
              the WMO stnId if the station is a WMO station.  It is a WMO station if
              digits 9-11="000".

          YEAR: 4 digit year of the station record.

          ELEMENT: element type, currently just "TAVG".

          VALUE: monthly value (MISSING=-9999)

          DMFLAG: data measurement flag, nine possible values:

                  blank = no measurement information applicable
                  a-i = number of days missing in calculation of monthly mean
                        temperature (currently only applies to the 1218 USHCN
                        V2 stations included within GHCNM)

          QCFLAG: quality control flag, seven possibilities within
                  quality controlled unadjusted (qcu) dataset, and 2
                  possibilities within the quality controlled adjusted (qca)
                  dataset.

                  Quality Controlled Unadjusted (QCU) QC Flags:

                  BLANK = no failure of quality control check or could not be
                          evaluated.

                  D = monthly value is part of an annual series of values that
                      are exactly the same (e.fldno. duplicated) within another
                      year in the station's record.

                  K = monthly value is part of a consecutive run (e.fldno. streak)
                      of values that are identical.  The streak must be >= 4
                      months of the same value.

                  L = monthly value is isolated in time within the station
                      record, and this is defined by having no immediate non-
                      missing values 18 months on either side of the value.

                  O = monthly value that is >= 5 bi-weight standard deviations
                      from the bi-weight mean.  Bi-weight statistics are
                      calculated from a series of all non-missing values in
                      the station's record for that particular month.

                  S = monthly value has failed spatial consistency check
                      (relative to their respective climatological means to
                       concurrent z-scores at the nearest 20 neighbors located
                       withing 500 km of the target; A temperature fails if
                       (i) its z-score differs from the regional (target and
                       neighbor) mean z-score by at least 3.5 standard
                       deviations and (ii) the target's temperature anomaly
                       differs by at least 2.5 deg C from all concurrent
                       temperature anomalies at the neighbors.

                  T = monthly value has failed temporal consistency check
                      (temperatures whose anomalies differ by more than
                      4 deg C from concurent anomalies at the five nearest
                      neighboring stations whose temperature anomalies are
                      well correlated with the target (correlation > 0.7 for
                      the corresponding calendar monthly).

                  W = monthly value is duplicated from the previous month,
                      based upon regional and spatial criteria and is only
                      applied from the year 2000 to the present.

                  Quality Controlled Adjusted (QCA) QC Flags:

                  M = values with a non-blank quality control flag in the "qcu"
                      dataset are set to missing the adjusted dataset and given
                      an "M" quality control flag.

                  X = pairwise algorithm removed the value because of too many
                      inhomogeneities.

          DSFLAG: data source flag for monthly value, 18 possibilities:

                  C = Monthly Climatic Data of the World (MCDW) QC completed
                      but value is not yet published

                  G = GHCNM v2 station, that was not a v2 station that had multiple
                      time series (for the same element).

                  K = received by the UK Met Office

                  M = Final (Published) Monthly Climatic Data of the World
                     (MCDW)

                  N = Netherlands, KNMI (Royal Netherlans Meteorological
                      Institute)

                  P = CLIMAT (Data transmitted over the GTS, not yet fully
                      processed for the MCDW)

                  U = USHCN v2

                  W = World Weather Records (WWR), 9th series 1991 through 2000

             0 to 9 = For any station originating from GHCNM v2 that had
                      multiple time series for the same element, this flag
                      represents the 12th digit in the ID from GHCNM v2.
                      See section 2.2.2 for additional information.

    2.2.2 STATIONS WITH MULTIPLE TIME SERIES

          The GHCNM v2 contained several thousand stations that had multiple
          time series of monthly mean temperature data.  The 12th digit of
          each data record, indicated the time series number, and thus there
          was a potential maximum of 10 time series (e.fldno. 0 through 9).  These
          same stations in v3 beta have undergone a merge process, to reduce
          the station time series to one single series, based upon these
          original and at most 10 time series.

          A simple algorithm was applied to perform the merge.  The algorithm
          consisted of first finding the length (based upon number of non
          missing observations) for each of the time series and then
          combining all of the series into one based upon a priority scheme
          that would "write" data to the series for the longest series last.

          Therefore, if station A, had 3 time series of TAVG data, as follows:

          1900 to 1978 (79 years of data) [series 1]
          1950 to 1985 (36 years of data) [series 2]
          1990 to 2007 (18 years of data) [series 3]

          The final series would consist of:

          1900 to 1978 [series 1]
          1979 to 1985 [series 2]
          1990 to 2007 [series 3]

          The original series number in GHCNM v2, is retained in the GHCNM v3
          beta data source flag.

          One caveat to this merge process, is that in the final GHCNM v3 beta
          processing there is still a master level construction process
          performed daily, where the entire dataset is construction according
          to a source order overwrite hiearchy (section 2.3), and it is
          possible that higher order data sources may be interspersed within
          the 3 series listed above.

    2.3 DATA SOURCE HIERARCHY

        The GHCNM v3 beta is reprocessed on a daily basis, which means as a
        part of that reprocessing, the dataset is reconstructed from all
        original sources. The advantage to this process is when source
        datasets are corrected and/or updated the inclusion into GHCNM v3
        beta is seemless.  The following sources (more fully described in
        section 2.2.1) have the following overwrite precedance within the
        daily reprocessing of GHCNM v3 (e.fldno. source K overwrites source P)

        P,K,G,U,0-9,C,N,M,W
  */

/*
  dat file
          1         2         3         4         5         6         7         8         9         10        11        12
0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
101603550001878TAVG  890  1  950  1 1110  1 1610  1 1980  1 2240  1 2490  1 2680  1 2320  1 2057E   1370  1 1150  1
101603550001932TAVG 1010  1  980  1-9999    1420  1 1840  1-9999    2290  1-9999    2440  1-9999   -9999   -9999

  see testRegexp.testGhcnm()

  match (\d{11})(\d{4})TAVG([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)(.)(.)([ \-\d]{5})(.)?(.)?(.)?.*
  against 101603550001932TAVG 1010  1  980  1-9999    1420  1 1840  1-9999    2290  1-9999    2440  1-9999   -9999   -9999

 matches = true
 group 1 == 10160355000
 group 2 == 1932
 group 3 ==  1010
 group 4 ==
 group 5 ==
 group 6 == 1
 group 7 ==   980
 group 8 ==
 group 9 ==
 group 10 == 1
 group 11 == -9999
 group 12 ==
 group 13 ==
 group 14 ==
 group 15 ==  1420
 group 16 ==
 group 17 ==
 group 18 == 1
 group 19 ==  1840
 group 20 ==
 group 21 ==
 group 22 == 1
 group 23 == -9999
 group 24 ==
 group 25 ==
 group 26 ==
 group 27 ==  2290
 group 28 ==
 group 29 ==
 group 30 == 1
 group 31 == -9999
 group 32 ==
 group 33 ==
 group 34 ==
 group 35 ==  2440
 group 36 ==
 group 37 ==
 group 38 == 1
 group 39 == -9999
 group 40 ==
 group 41 ==
 group 42 ==
 group 43 == -9999
 group 44 ==
 group 45 ==
 group 46 ==
 group 47 == -9999
 group 48 == null
 group 49 == null
 group 50 == null
   */

  /*
inv file
          1         2         3         4         5         6         7         8         9         10        11        12
0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890

10160475000  35.4800    8.1300  813.0 TEBESSA                         818U   67MVxxno-9A 2WARM FOR./FIELD B
10160490000  35.6000   -0.6000   90.0 ORAN/ES SENIA       ALGERIA      98U  492HIxxCO10A 6WARM CROPS      B

match (\d{11}) ([ \.\-\d]{8}) ([ \.\-\d]{9}) ([ \.\-\d]{6}) (.{30}) (.{4})(.) (.{4})(..)(..)(..)(..)(.)(..)(.{15})(.)* against 10160490000  35.6000   -0.6000   90.0 ORAN/ES SENIA       ALGERIA      98U  492HIxxCO10A 6WARM CROPS      B

 matches = true
 group 1 == 10160490000
 group 2 ==  35.6000
 group 3 ==   -0.6000
 group 4 ==   90.0
 group 5 == ORAN/ES SENIA       ALGERIA
 group 6 ==   98
 group 7 == U
 group 8 ==  492
 group 9 == HI
 group 10 == xx
 group 11 == CO
 group 12 == 10
 group 13 == A
 group 14 ==  6
 group 15 == WARM CROPS
 group 16 == B
   */

public class Ghcnm2 extends AbstractIOServiceProvider {
  private static final String dataPatternRegexp =
          "(\\d{11})(\\d{4})TAVG([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)"+
          "([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)"+
          "([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)(.)(.)([ \\-\\d]{5})(.)?(.)?(.)?.*";

  private static final String stnPatternRegexp =
    "(\\d{11}) ([ \\.\\-\\d]{8}) ([ \\.\\-\\d]{9}) ([ \\.\\-\\d]{6}) (.{30}) ([ \\-\\d]{4})(.)([ \\-\\d]{5})(..)(..)(..)([ \\-\\d]{2})(.)(..)(.{16})(.).*";

  private static final Pattern dataPattern = Pattern.compile(dataPatternRegexp);
  private static final Pattern stnPattern = Pattern.compile(stnPatternRegexp);

  private static final String STNID = "stnid";

  private static final String STN_EXT = ".inv";
  private static final String DAT_EXT = ".dat";
  private static final String IDX_EXT = ".ncx";
  private static final String MAGIC_START_IDX = "GhncmIndex";
  private static final int version = 1;

  @Override
  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    String dataFile = raf.getLocation();
    int pos = dataFile.lastIndexOf(".");
    if (pos <= 0) return false;
    String base = dataFile.substring(0, pos);
    String ext = dataFile.substring(pos);

     // must be data file or station or index file
    if (!ext.equals(DAT_EXT) && !ext.equals(STN_EXT)&& !ext.equals(IDX_EXT))
      return false;

    if (ext.equals(IDX_EXT)) {
      // data, stn files must be in the same directory
      File datFile = new File(base+DAT_EXT);
      if (!datFile.exists())
         return false;
      File stnFile = new File(base+STN_EXT);
      if (!stnFile.exists())
        return false;

      raf.seek(0);
      String test = raf.readString(MAGIC_START_IDX.length());
      return test.equals(MAGIC_START_IDX);

    } else if (ext.equals(DAT_EXT)) {
      // stn file must be in the same directory
      File stnFile = new File(base+STN_EXT);
      return stnFile.exists() && isValidFile(raf, dataPattern);

    } else {
      // dat file must be in the same directory
      File stnFile = new File(base+DAT_EXT);
      return stnFile.exists() && isValidFile(raf, stnPattern);
     }
  }

  private boolean isValidFile(RandomAccessFile raf, Pattern p) throws IOException {
    raf.seek(0);
    String line;
    while (true) {
      line = raf.readLine();
      if (line == null) break;
      if (line.startsWith("#")) continue;
      if (line.trim().length() == 0) continue;
      Matcher matcher = p.matcher(line);
      return matcher.matches();
    }
    return false;
  }

  @Override
  public String getFileTypeId() {
    return "GHCNM";
  }

  @Override
  public String getFileTypeDescription() {
    return "GLOBAL HISTORICAL CLIMATOLOGY NETWORK MONTHLY";
  }

  @Override
  public String getFileTypeVersion() {
    return Integer.toString(version);
  }

  /////////////////////////////////////////////////////////////////////////
  private RandomAccessFile stnRaf, dataRaf;
  private HashMap<Long, StationIndex> map = new HashMap<Long, StationIndex>(10000);
  private int stn_fldno;
  private StructureDataRegexp.Vinfo dataVinfo, stnVinfo;

  @Override
  public void open(RandomAccessFile raff, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    String dataFile = raff.getLocation();
    int pos = dataFile.lastIndexOf(".");
    String base = dataFile.substring(0, pos);
    String ext = dataFile.substring(pos);

    if (ext.equals(IDX_EXT)) {
      dataRaf = RandomAccessFile.acquire(base+DAT_EXT);
      stnRaf = RandomAccessFile.acquire(base+STN_EXT);

     } else if (ext.equals(DAT_EXT)) {
      dataRaf = raff;
      stnRaf = RandomAccessFile.acquire(base+STN_EXT);

     } else {
      stnRaf = raff;
      dataRaf = RandomAccessFile.acquire(base+DAT_EXT);
     }

    NcmlConstructor ncmlc = new NcmlConstructor();
    if (!ncmlc.populateFromResource("resources/nj22/iosp/ghcnm.ncml", ncfile)) {
      throw new IllegalStateException(ncmlc.getErrlog().toString());
    }
    ncfile.finish();

    dataVinfo = setVinfo(dataRaf, ncfile, dataPattern, "all_data");
    stnVinfo = setVinfo(stnRaf, ncfile, stnPattern, "station");

    StructureMembers.Member m = stnVinfo.sm.findMember(STNID);
    StructureDataRegexp.VinfoField f = (StructureDataRegexp.VinfoField) m.getDataObject();
    stn_fldno = f.fldno;

     // make index file if needed
    File idxFile = new File(base+IDX_EXT);
    if (!idxFile.exists())
      makeIndex(stnVinfo, dataVinfo, idxFile);
    else
      readIndex(idxFile.getPath());
  }

  private StructureDataRegexp.Vinfo setVinfo(RandomAccessFile raff, NetcdfFile ncfile, Pattern p, String seqName) {
    Sequence seq = (Sequence) ncfile.findVariable(seqName);
    StructureMembers sm = seq.makeStructureMembers();
    StructureDataRegexp.Vinfo result = new StructureDataRegexp.Vinfo(raff, sm, p);
    seq.setSPobject(result);

    int fldno = 1;
    for (StructureMembers.Member m : sm.getMembers()) {
      StructureDataRegexp.VinfoField vf = new StructureDataRegexp.VinfoField(fldno++);
      Variable v = seq.findVariable(m.getName());
      Attribute att = v.findAttribute("iosp_scale");
      if (att != null) {
        vf.hasScale = true;
        vf.scale = att.getNumericValue().floatValue();
        v.remove(att);
      }
      m.setDataObject( vf);
    }

    return result;
  }

  public void close() throws java.io.IOException {
    stnRaf.close();
    dataRaf.close();
  }

  ////////////////////////////////////////////////////////////////////

  @Override
  public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
    StructureDataRegexp.Vinfo vinfo = (StructureDataRegexp.Vinfo) v2.getSPobject();
    return new ArraySequence( vinfo.sm, new SeqIter(vinfo), vinfo.nelems);
  }

  @Override
  public StructureDataIterator getStructureIterator(Structure s, int bufferSize) throws java.io.IOException {
    StructureDataRegexp.Vinfo vinfo = (StructureDataRegexp.Vinfo) s.getSPobject();
    return new SeqIter(vinfo);
  }

  private class SeqIter implements StructureDataIterator {
    private StructureDataRegexp.Vinfo vinfo;
    private long bytesRead;
    private long totalBytes;
    private int recno;
    private StructureData curr;

    SeqIter(StructureDataRegexp.Vinfo vinfo) throws IOException {
      this.vinfo = vinfo;
      totalBytes = (int) vinfo.rafile.length();
      vinfo.rafile.seek(0);
    }

    @Override
    public StructureDataIterator reset() {
      bytesRead = 0;
      recno = 0;

      try {
        vinfo.rafile.seek(0);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return this;
    }

    @Override
    public boolean hasNext() throws IOException {
      boolean more = (bytesRead < totalBytes); // && (recno < 10);
      if (!more) {
        vinfo.nelems = recno;
        //System.out.printf("nelems=%d%n", recno);
        return false;
      }
      curr = reallyNext();
      more = (curr != null);
      if (!more) {
        vinfo.nelems = recno;
        //System.out.printf("nelems=%d%n", recno);
        return false;
      }
      return more;
    }

    @Override
    public StructureData next() throws IOException {
      return curr;
    }

    private StructureData reallyNext() throws IOException {
      Matcher matcher;
      while (true) {
        String line = vinfo.rafile.readLine();
        if (line == null) return null;
        if (line.startsWith("#")) continue;
        if (line.trim().length() == 0) continue;
        matcher = vinfo.p.matcher(line);
        if (matcher.matches())
          break;
        System.out.printf("FAIL %s%n", line);
      }
      //System.out.printf("%s%n", line);
      bytesRead = vinfo.rafile.getFilePointer();
      recno++;
      return new StructureDataRegexpGhcnm(vinfo.sm, matcher);
    }

    @Override
    public int getCurrentRecno() {
      return recno - 1;
    }

  }

  //////////////////////////////////////////////////////

  private class StructureDataRegexpGhcnm extends StructureDataRegexp {
    // StructureMembers members;
    // Matcher matcher;          // matcher on the station ascii

    StructureDataRegexpGhcnm(StructureMembers members, Matcher matcher) {
      super(members, matcher);
    }

    @Override
    // nested array sequence must be the stn_data
    public ArraySequence getArraySequence(StructureMembers.Member m) {
      String svalue = matcher.group(stn_fldno).trim();
      Long stnId = Long.parseLong(svalue); // extract the station id
      StationIndex si = map.get(stnId); // find its index
      return new ArraySequence(dataVinfo.sm, new StnDataIter(dataVinfo.sm, si), -1);
    }
  }

  private class StnDataIter implements StructureDataIterator {
    private StructureMembers sm;
    private int countRead;
    private StationIndex stationIndex;

    StnDataIter(StructureMembers sm, StationIndex stationIndex) {
      this.sm = sm;
      this.stationIndex = stationIndex;
      reset();
    }

    @Override
    public StructureDataIterator reset() {
      countRead = 0;
      try {
        dataRaf.seek(stationIndex.dataPos);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return this;
    }

    @Override
    public boolean hasNext() throws IOException {
      return (countRead < stationIndex.dataCount);
    }

    @Override
    public StructureData next() throws IOException {
      Matcher matcher;
      String line;
      while (true) {
        line = dataRaf.readLine();
        if (line == null) return null;
        if (line.startsWith("#")) continue;
        if (line.trim().length() == 0) continue;
        matcher = dataPattern.matcher(line);
        if (matcher.matches())
          break;
      }
      //System.out.printf("%s%n", line);
      countRead++;
      return new StructureDataRegexp(sm, matcher);
    }

    @Override
    public int getCurrentRecno() {
      return countRead - 1;
    }
  }

    ///////////////////////////////////////////

  private void readIndex(String indexFilename) throws IOException {
    try (FileInputStream fin = new FileInputStream(indexFilename)) {

      if (!NcStream.readAndTest(fin, MAGIC_START_IDX.getBytes(CDM.utf8Charset)))
        throw new IllegalStateException("bad index file");
      int version = fin.read();
      if (version != 1)
        throw new IllegalStateException("Bad version = " + version);

      int count = NcStream.readVInt(fin);

      for (int i = 0; i < count; i++) {
        int size = NcStream.readVInt(fin);
        byte[] pb = new byte[size];
        NcStream.readFully(fin, pb);
        StationIndex si = decodeStationIndex(pb);
        map.put(si.stnId, si);
      }
    }

    System.out.println(" read index map size=" + map.values().size());
  }

 private void makeIndex(StructureDataRegexp.Vinfo stnInfo, StructureDataRegexp.Vinfo dataInfo, File indexFile ) throws IOException {
    // get map of Stations
    StructureMembers.Member m = stnInfo.sm.findMember(STNID);
    StructureDataRegexp.VinfoField f = (StructureDataRegexp.VinfoField) m.getDataObject();
    int stnCount = 0;

    // read through entire file LOOK: could use SeqIter
    stnInfo.rafile.seek(0);
    while (true) {
      long stnPos = stnInfo.rafile.getFilePointer();
      String line = stnInfo.rafile.readLine();
      if (line == null) break;

      Matcher matcher = stnInfo.p.matcher(line);
      if (!matcher.matches()) {
        System.out.printf("FAIL %s%n", line);
        continue;
      }
      String svalue = matcher.group(f.fldno);
      Long id = Long.parseLong(svalue.trim());

      StationIndex s = new StationIndex();
      s.stnId = id;
      s.stnPos = stnPos;
      map.put(id, s);
      stnCount++;
    }

    // assumes that the stn data is in order by stnId
    m = dataInfo.sm.findMember(STNID);
    f = (StructureDataRegexp.VinfoField) m.getDataObject();
    StationIndex currStn = null;
    int totalCount = 0;

    // read through entire data file
    dataInfo.rafile.seek(0);
    while (true) {
      long dataPos = dataInfo.rafile.getFilePointer();
      String line = dataInfo.rafile.readLine();
      if (line == null) break;

      Matcher matcher = dataInfo.p.matcher(line);
      if (!matcher.matches()) {
        System.out.printf("FAIL %s%n", line);
        continue;
      }

      String svalue = matcher.group(f.fldno).trim();
      Long id = Long.parseLong(svalue);

      if ((currStn == null) || (currStn.stnId != id)) {
        StationIndex s = map.get(id);
        if (s == null)
          System.out.printf("Cant find %d%n", id);
        else if (s.dataCount != 0)
          System.out.printf("Not in order %d at pos %d %n", id, dataPos);
        else {
          s.dataPos = dataPos;
          totalCount++;
        }
        currStn = s;
      }
      if (currStn != null) currStn.dataCount++;
    }
    //System.out.printf("ok stns=%s data=%d%n", stnCount, totalCount);

    //////////////////////////////
    // write the index file
    try (FileOutputStream fout = new FileOutputStream(indexFile)) { // LOOK need DiskCache for non-writeable directories
      long size = 0;

      //// header message
      fout.write(MAGIC_START_IDX.getBytes(CDM.utf8Charset));
      fout.write(version);
      size += NcStream.writeVInt(fout, stnCount);

    /* byte[] pb = encodeStationListProto( map.values());
    size += NcStream.writeVInt(fout, pb.length);
    size += pb.length;
    fout.write(pb); */

      for (StationIndex s : map.values()) {
        byte[] pb = s.encodeStationProto();
        size += NcStream.writeVInt(fout, pb.length);
        size += pb.length;
        fout.write(pb);
      }
    }

    //System.out.println(" index size=" + size);
  }

  private StationIndex decodeStationIndex(byte[] data) throws InvalidProtocolBufferException {
    ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto = GhcnmProto.StationIndex.parseFrom(data);
    return new StationIndex(proto);
  }

  private static class StationIndex {
    long stnId;
    long stnPos; // file pos in inv file
    long dataPos; // file pos of first data line in the data file
    int dataCount; // number of data records

    StationIndex() {
    }

    StationIndex(ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto) {
      this.stnId = proto.getStnid();
      this.stnPos = proto.getStnPos();
      this.dataPos = proto.getDataPos();
      this.dataCount = proto.getDataCount();
    }

    private byte[] encodeStationProto()  {
      GhcnmProto.StationIndex.Builder builder = GhcnmProto.StationIndex.newBuilder();
      builder.setStnid(stnId);
      builder.setStnPos(stnPos);
      builder.setDataPos(dataPos);
      builder.setDataCount(dataCount);
      ucar.nc2.iosp.noaa.GhcnmProto.StationIndex proto =  builder.build();
      return proto.toByteArray();
    }
  }

}


/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2010, OPeNDAP, Inc.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
//
// Author: James Gallagher <jgallagher@opendap.org>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms,
// with or without modification, are permitted provided
// that the following conditions are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// - Neither the name of the OPeNDAP nor the names of its contributors may
//   be used to endorse or promote products derived from this software
//   without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////////////////////////////////////////////////

package opendap.dap;

import opendap.dap.parsers.Dap2Parser;
import opendap.dap.parsers.ParseException;

import java.io.*;

//import opendap.dap.parsers.ErrorParser;

/**
 * Holds an exception thrown by OPeNDAP server to a client.
 *

 * Unlike the other OPeNDAP exceptions, this one contains extra methods to
 * get the various fields sent by the server, and a parse method
 * to parse the Error sent from the server.
 *

 *

 * This class will be changing it's name to opendap.dap.DAP2Exception.
 * I expect that it will be deprecated in the next release.
 *
You've been warned.
 Questions? Ask ndp@opendap.org.
 *

 *
 * @author jehamby
 * @version $Revision: 15901 $
 */
public class DAP2Exception extends Exception
{

    /**
     * Undefined error.
     */
    public static final int UNDEFINED_ERROR = -1;
    /**
     * Unknown error.
     */
    public static final int UNKNOWN_ERROR = 0;
    /**
     * The file specified by the OPeNDAP URL does not exist.
     */
    public static final int NO_SUCH_FILE = 1;
    /**
     * The variable specified in the OPeNDAP URL does not exist.
     */
    public static final int NO_SUCH_VARIABLE = 2;
    /**
     * The expression specified in the OPeNDAP URL is not valid.
     */
    public static final int MALFORMED_EXPR = 3;
    /**
     * The user has no authorization to read the OPeNDAP URL.
     */
    public static final int NO_AUTHORIZATION = 4;
    /**
     * The file specified by the OPeNDAP URL can not be read.
     */
    public static final int CANNOT_READ_FILE = 5;

    /*
    * Some Error objects may contain programs which can be used to
    * correct the reported error. These programs are run using a public
    * member function of the Error class. If an Error object does not
    * have an associated correction program, the program type is NO_PROGRAM.
    */

    /**
     * Undefined program type.
     */
    public static final int UNDEFINED_PROG_TYPE = -1;
    /**
     * This Error does not contain a program.
     */
    public static final int NO_PROGRAM = 0;
    /**
     * This Error contains Java bytecode.
     */
    public static final int JAVA_PROGRAM = 1;
    /**
     * This Error contains TCL code.
     */
    public static final int TCL_PROGRAM = 2;

    /**
     * The error code.
     *
     * @serial
     */
    private int errorCode;
    /**
     * The error message.
     *
     * @serial
     */
    private String errorMessage;
    /**
     * The program type.
     *
     * @serial
     */
    private int programType;

    /**
     * The program source.  if programType is TCL_PROGRAM, then this is ASCII
     * text.  Otherwise, undefined (this will need to become a byte[] array if
     * the server sends Java bytecodes, for example).
     *
     * @serial
     */
    private String programSource;

    /**
     * Construct an empty DAP2Exception.
     */
    public DAP2Exception()
    {
        // this should never be seen, since this class overrides getMessage()
        // to display its own error message.
        super("DAP2Exception");
    }

    /**
     * Construct a DAP2Exception.
     */
    public DAP2Exception(String msg)
    {
        this();
        errorCode = UNKNOWN_ERROR;
        errorMessage = msg;
    }

    /**
     * Construct a DAP2Exception with the given message.
     *
     * @param code the error core
     * @param msg  the error message
     */
    public DAP2Exception(int code, String msg)
    {
        this();
        errorCode = code;
        errorMessage = msg;
    }

    /**
     * Construct a DAP2Exception with the given causal exception.
     *
     * @param cause the causing exception
     */
    public DAP2Exception(Exception cause)
    {
        super(cause);
        errorCode = 0;
        errorMessage = cause.toString();
    }

    /**
     * Construct a DAP2Exception with the given causal exception.
     *
     * @param cause the causing exception
     */
    public DAP2Exception(int code, Exception cause)
    {
        this(cause);
        errorCode = code;
    }

    /**
     * Returns the error code.
     *
     * @return the error code.
     */
    public final int getErrorCode()
    {
        return errorCode;
    }

    /**
     * Returns the error message.
     *
     * @return the error message.
     */
    public final String getErrorMessage()
    {
        return errorMessage;
    }

    /**
     * Returns the program type.
     *
     * @return the program type.
     */
    public final int getProgramType()
    {
        return programType;
    }

    /**
     * Returns the program source.
     *
     * @return the program source.
     */
    public final String getProgramSource()
    {
        return programSource;
    }

    /**
     * Returns the detail message of this throwable object.
     *
     * @return the detail message of this throwable object.
     */
    //Coverity[CALL_SUPER]
    public String getMessage()
    {
        return errorMessage;
    }

    /**
     * Sets the error code.
     *
     * @param code the error code.
     */
    public final void setErrorCode(int code)
    {
        errorCode = code;
    }

    /**
     * Sets the error message.
     *
     * @param msg the error message.
     */
    public final void setErrorMessage(String msg)
    {
        errorMessage = msg;
    }

    /**
     * Sets the program type.
     *
     * @param type the program type.
     */
    public final void setProgramType(int type)
    {
        programType = type;
    }

    /**
     * Sets the program source.
     *
     * @param source the program source.
     */
    public final void setProgramSource(String source)
    {
        programSource = source;
    }

    /**
     * Reads an Error description from the named InputStream.  This
     * method calls a generated parser to interpret an ASCII representation of an
     * Error, and regenerate it as a DAP2Exception.
     *
     * @param stream the text containing the Error to parse.
     */
    public final boolean parse(InputStream stream)
    {
        Dap2Parser parser = new Dap2Parser(new DefaultFactory());
        String text;
        try {
            text = DConnect2.captureStream(stream);
            if(parser.errparse(text, this) != Dap2Parser.DapERR) return false;
        } catch (ParseException pe) {
            this.initCause(pe);
        } catch (IOException pe) {
            this.initCause(pe);
        }
        return true;
    }

    /**
     * Print the Error message on the given PrintWriter.
     * This code can be used by servlets to throw DAP2Exception to client.
     *
     * @param os the PrintWriter to use for output.
     */
    public void print(PrintWriter os)
    {
        os.println("Error {");
        os.println("    code = " + errorCode + ";");
        if(errorMessage != null) os.println("    message = " + dumpword(errorMessage) + ";");
        //if(programType > 0) os.println("    program_type = " + programType + ";");
        if(programSource != null) os.println("    program = " + dumpword(programSource) + ";");
        os.println("};");
    }

    private String dumpword(String word)
    {
        if(word == null)
            return "";
        else if(word.charAt(0) == '"')
            return word;
        else
            return "\"" + word + "\"";
    }

    /**
     * Print the Error message on the given OutputStream.
     *
     * @param os the OutputStream to use for output.
     * @see DAP2Exception#print(PrintWriter)
     */
    public final void print(OutputStream os)
    {
        PrintWriter pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(os, Util.UTF8)));
        print(pw);
        pw.flush();
    }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ui.grib;

import ucar.nc2.grib.GribResourceReader;
import ucar.nc2.grib.grib1.tables.Grib1Customizer;
import ucar.nc2.grib.grib1.Grib1Parameter;
import ucar.nc2.grib.grib1.tables.Grib1ParamTableReader;
import ucar.nc2.grib.grib1.tables.Grib1ParamTables;
import ucar.nc2.ui.dialog.Grib1TableCompareDialog;
import ucar.nc2.ui.widget.*;
import ucar.nc2.ui.widget.IndependentWindow;
import ucar.nc2.ui.dialog.Grib1TableDialog;
import ucar.nc2.ui.widget.PopupMenu;
import ucar.nc2.ui.widget.TextHistoryPane;
import ucar.nc2.units.SimpleUnit;
import ucar.nc2.util.IO;
import ucar.nc2.wmo.CommonCodeTable;
import ucar.nc2.wmo.Util;
import ucar.util.prefs.PreferencesExt;
import ucar.util.prefs.ui.BeanTable;

import javax.swing.*;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.List;

/**
 * Show Grib1 Tables
 *
 * @author caron
 * @since Sep 26, 2010
 */
public class Grib1TablesViewer extends JPanel {

  private PreferencesExt prefs;

  private BeanTable codeTable, entryTable;
  private JSplitPane split;

  private TextHistoryPane infoTA;
  private IndependentWindow infoWindow;

  private Grib1TableDialog showTableDialog;
  private Grib1TableCompareDialog compareTableDialog;

  public Grib1TablesViewer(final PreferencesExt prefs, JPanel buttPanel) {
    this.prefs = prefs;

    codeTable = new BeanTable(TableBean.class, (PreferencesExt) prefs.node("CodeTableBean"), false);
    codeTable.addListSelectionListener(e -> {
      TableBean csb = (TableBean) codeTable.getSelectedBean();
      setEntries(csb.table);
    });

    ucar.nc2.ui.widget.PopupMenu varPopup = new PopupMenu(codeTable.getJTable(), "Options");
    varPopup.addAction("Show File contents", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        TableBean bean = (TableBean) codeTable.getSelectedBean();
        if (bean == null) return;
        showFile(bean);
      }
    });

    varPopup.addAction("Compare to default WMO table", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        TableBean bean = (TableBean) codeTable.getSelectedBean();
        if (bean == null) return;
        initTableDialog();

        Grib1ParamTableReader wmo = new Grib1ParamTables().getParameterTable(0, 0, bean.getVersion());
        if (wmo == null) {
          infoTA.setText("Cant find WMO version " + bean.getVersion());
          infoWindow.show();
          return;
        }

        compareTableDialog.setTable1(new TableBean(wmo));
        compareTableDialog.setTable2(bean);
        compareTableDialog.setVisible(true);
      }
    });

    varPopup.addAction("Compare two tables", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        TableBean bean = (TableBean) codeTable.getSelectedBean();
        if (bean == null) return;
        initTableDialog();

        List list = codeTable.getSelectedBeans();
        if (list.size() == 2) {
          TableBean bean1 = (TableBean) list.get(0);
          TableBean bean2 = (TableBean) list.get(1);
          compareTableDialog.setTable1(bean1);
          compareTableDialog.setTable2(bean2);
          compareTableDialog.setVisible(true);
        }
      }
    });

    varPopup.addAction("Compare to all non-local tables", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        TableBean bean = (TableBean) codeTable.getSelectedBean();
        if (bean == null) return;
        initTableDialog();

        compareTableDialog.setTable1(bean);
        compareTableDialog.setTable2(null);
        compareTableDialog.setVisible(true);
      }
    });

    entryTable = new BeanTable(EntryBean.class, (PreferencesExt) prefs.node("EntryBean"), false);
    /* entryTable.addListSelectionListener(new ListSelectionListener() {
      public void valueChanged(ListSelectionEvent e) {
        entryTable.getSelectedBean();
      }
    }); */

    varPopup = new PopupMenu(entryTable.getJTable(), "Options");
    varPopup.addAction("Show in all tables", new AbstractAction() {
      public void actionPerformed(ActionEvent e) {
        EntryBean bean = (EntryBean) entryTable.getSelectedBean();
        if (bean == null) return;
        showAll(bean);
      }
    });

    // the info window
    infoTA = new TextHistoryPane();
    infoWindow = new IndependentWindow("Extra Information", BAMutil.getImage("netcdfUI"), infoTA);
    infoWindow.setBounds((Rectangle) prefs.getBean("InfoWindowBounds", new Rectangle(300, 300, 800, 600)));

    split = new JSplitPane(JSplitPane.VERTICAL_SPLIT, false, codeTable, entryTable);
    split.setDividerLocation(prefs.getInt("splitPos", 500));

    setLayout(new BorderLayout());
    add(split, BorderLayout.CENTER);

    AbstractButton infoButton = BAMutil.makeButtcon("Information", "Show Table Used", false);
    infoButton.addActionListener(e -> {
        if (showTableDialog == null) {
          showTableDialog = new Grib1TableDialog((Frame) null);
          showTableDialog.pack();
        }
        showTableDialog.setVisible(true);
    });
    buttPanel.add(infoButton);

    try {
      List<Grib1ParamTableReader> tables = Grib1ParamTables.getStandardParameterTables();
      java.util.List<TableBean> beans = new ArrayList<>(tables.size());
      for (Grib1ParamTableReader t : tables) {
        beans.add(new TableBean(t));
      }
      //Collections.sort(beans);
      codeTable.setBeans(beans);

    } catch (Exception e) {
      e.printStackTrace();
    }

  }

  private void initTableDialog() {
    if (compareTableDialog == null) {
      compareTableDialog = new Grib1TableCompareDialog((Frame) null);
      compareTableDialog.pack();
      compareTableDialog.addPropertyChangeListener("OK",
          evt -> compareTables((Grib1TableCompareDialog.Data) evt.getNewValue()));
    }
  }

  public void save() {
    codeTable.saveState(false);
    entryTable.saveState(false);
    prefs.putBeanObject("InfoWindowBounds", infoWindow.getBounds());
    //prefs.putBeanObject("InfoWindowBounds2", infoWindow2.getBounds());
    prefs.putInt("splitPos", split.getDividerLocation());
    //prefs.putInt("splitPos2", split2.getDividerLocation());
    // if (fileChooser != null) fileChooser.save();
  }

  public void setTable(String filename) throws IOException {
    Grib1ParamTableReader table = new Grib1ParamTableReader(filename);
    TableBean bean = new TableBean(table);
    codeTable.addBean( bean);
    codeTable.setSelectedBean(bean);
  }

  private void setEntries(Grib1ParamTableReader table) {
    Map<Integer, Grib1Parameter> map = table.getParameters();
    if (map == null) {
      map = table.getParameters();
    }

    ArrayList<Integer> params = new ArrayList<>();
    params.addAll(map.keySet());
    Collections.sort(params);
    java.util.List<EntryBean> beans = new ArrayList<>(params.size());
    for (Integer key : params) {
      beans.add(new EntryBean( map.get(key)));
    }
    entryTable.setBeans(beans);
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////

  private void showFile(TableBean bean) {
    infoTA.setText("Table:" + bean.getPath() + "\n");
    try (InputStream is = GribResourceReader.getInputStream(bean.getPath())) {
      infoTA.appendLine( IO.readContents(is));
      infoWindow.setVisible(true);

    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  private void showAll(EntryBean ebean) {
    Formatter f = new Formatter();
    infoTA.setText("Entry:" + ebean.getNumber() + "\n");
    for (Object bean : codeTable.getBeans()) {
      TableBean tbean = (TableBean) bean;
      Grib1Parameter p = tbean.table.getLocalParameter(ebean.getNumber());
      if (p != null) f.format(" %s from %s (%d)%n", p, tbean.table.getName(), tbean.table.getParameters().hashCode());
    }

    infoTA.appendLine(f.toString());
    infoWindow.setVisible(true);
  }

  private void compareTables(Grib1TableCompareDialog.Data data) {
    Formatter f = new Formatter();
    if ( data.table2bean == null)
      compareAll(data.table1bean.table, data, f);
    else
      compare(data.table1bean.table, data.table2bean.table, data, f);

    infoTA.setText(f.toString());
    infoTA.gotoTop();
    infoWindow.show();
  }

  private void compare(Grib1ParamTableReader t1, Grib1ParamTableReader t2, Grib1TableCompareDialog.Data data, Formatter out) {
    out.format("Compare%n %s%n %s%n", t1.toString(), t2.toString());
    Map<Integer, Grib1Parameter> h1 = t1.getParameters();
    Map<Integer, Grib1Parameter> h2 = t2.getParameters();
    List<Integer> keys = new ArrayList<>(h1.keySet());
    Collections.sort(keys);

    for (Integer key : keys) {
      Grib1Parameter d1 = h1.get(key);
      Grib1Parameter d2 = h2.get(key);
      if (d2 == null) {
        if (data.showMissing) out.format("**No key %s (%s) in second table%n", key, d1);
      } else {
        if (data.compareDesc) {
          if (!equiv(d1.getDescription(), d2.getDescription()))
            out.format(" %s desc%n   %s%n   %s%n", d1.getNumber(), d1.getDescription(), d2.getDescription());
        }
        if (data.compareNames) {
          if (!equiv(d1.getName(), d2.getName()))
            out.format(" %d name%n   %s%n   %s%n", d1.getNumber(), d1.getName(), d2.getName());
        }
        if (data.compareUnits) {
          if (!equiv(d1.getUnit(), d2.getUnit()))
            out.format(" %s units%n   %s%n   %s%n", d1.getNumber(), d1.getUnit(), d2.getUnit());
        }
        if (data.cleanUnits) {
          String cu1 =  Util.cleanUnit(d1.getUnit());
          String cu2 =  Util.cleanUnit(d2.getUnit());
          if (!equiv(cu1, cu2)) out.format(" %s cleanUnits%n   %s%n   %s%n", d1.getNumber(), cu1, cu2);
        }
        if (data.udunits) {
          String cu1 =  Util.cleanUnit(d1.getUnit());
          String cu2 =  Util.cleanUnit(d2.getUnit());
            try {
              SimpleUnit su1 = SimpleUnit.factoryWithExceptions(cu1);
              if (!su1.isCompatible(cu2))
                out.format(" %s udunits%n   %s%n   %s%n", d1.getNumber(), cu1, cu2);
            } catch (Exception e) {
              out.format(" %s udunits%n   cant parse = %s%n   %s%n", d1.getNumber(), cu1, cu2);
            }
        }
      }
    }

    if (data.showMissing) {
      out.format("%n***Check if entries are missing in first table%n");
      keys = new ArrayList<>(h2.keySet());
      Collections.sort(keys);
      for (Integer key : keys) {
        Grib1Parameter d1 = h1.get(key);
        Grib1Parameter d2 = h2.get(key);
        if (d1 == null)
          out.format("**No key %s (%s) in first table%n", key, d2);
      }
    }

  }

  private boolean equiv(String org1, String org2) {
    if (org1 == org2) return true;
    if (org1 == null) return false;
    if (org2 == null) return false;
    return org1.equalsIgnoreCase(org2);
  }

  private void compareAll(Grib1ParamTableReader t1, Grib1TableCompareDialog.Data data, Formatter out) {
    out.format("Compare All non-local Tables%s%n", t1.toString());
    Map<Integer, Grib1Parameter> h1 = t1.getParameters();
    List<Integer> keys = new ArrayList<>(h1.keySet());
    Collections.sort(keys);

    for (Integer key : keys) {
      Grib1Parameter d1 = h1.get(key);
      out.format("%n--- %s%n", d1);

      for (Object bean : codeTable.getBeans()) {
        TableBean tbean = (TableBean) bean;
        if (tbean.getVersion() > 127) continue;

        Grib1Parameter d2 = tbean.table.getLocalParameter(d1.getNumber());
        if (d2 != null) {
          boolean descDiff = data.compareDesc &&  !equiv(d1.getDescription(), d2.getDescription());
          boolean namesDiff = data.compareNames &&  !equiv(d1.getName(), d2.getName());
          boolean unitsDiff = data.compareUnits &&  !equiv(d1.getUnit(), d2.getUnit());
          boolean cunitsDiff = data.cleanUnits &&  !equiv(Util.cleanUnit(d1.getUnit()), Util.cleanUnit(d2.getUnit()));
          boolean udunitsDiff = false;
          if (data.udunits) {
            String cu1 =  Util.cleanUnit(d1.getUnit());
            String cu2 =  Util.cleanUnit(d2.getUnit());
              try {
                SimpleUnit su1 = SimpleUnit.factoryWithExceptions(cu1);
                udunitsDiff = !su1.isCompatible(cu2);
              } catch (Exception e) {
                udunitsDiff = false;
              }
          }

          if (descDiff)
            out.format("    desc=%s from %s%n", d2.getDescription(), tbean.table.getPath());
          if (namesDiff)
            out.format("    name=%s from %s%n", d2.getName(), tbean.table.getPath());
          if (unitsDiff || cunitsDiff || udunitsDiff )
            out.format("    units=%s from %s%n", d2.getUnit(), tbean.table.getPath());
        }
      }
    }

  }

  public class TableBean implements Comparable<TableBean> {
    Grib1ParamTableReader table;

    // no-arg constructor

    public TableBean() {
    }

    // create from a dataset

    public TableBean(Grib1ParamTableReader table) {
      this.table = table;
    }

    public int getCenter_id() {
      return table.getCenter_id();
    }

    public String getCenter() {
      return CommonCodeTable.getCenterName(table.getCenter_id(), 1);
    }

    public String getSubCenter() {
      return Grib1Customizer.getSubCenterNameStatic(table.getCenter_id(), table.getSubcenter_id());
    }

    public int getSubcenter_id() {
      return table.getSubcenter_id();
    }

    public int getVersion() {
      return table.getVersion();
    }

    public String getPath() {
      return table.getPath();
    }

    public String getName() {
      return table.getName();
    }

    public int getKey() {
      return table.getKey();
    }

    @Override
    public int compareTo(TableBean o) {
      int ret = getCenter_id() - o.getCenter_id();
      if (ret == 0)
        ret = getSubcenter_id() - o.getSubcenter_id();
      if (ret == 0)
        ret = getVersion() - o.getVersion();
      return  ret;
    }
  }

  public class EntryBean {
    Grib1Parameter param;

    // no-arg constructor
    public EntryBean() {
    }

    public EntryBean(Grib1Parameter param) {
      this.param = param;
    }

    public String getName() {
      return param.getName();
    }

    public String getCFname() {
      return param.getCFname();
    }

    public String getWMOdesc() {
      Grib1ParamTableReader wmo = Grib1ParamTables.getDefaultWmoTable();
      Grib1Parameter p = wmo.getParameter(param.getNumber());
      return p == null ? "" : p.getDescription();
    }

    public String getDescription() {
      return param.getDescription();
    }

    public String getUnit() {
      return param.getUnit();
    }

    public String getCleanUnit() {
      return Util.cleanUnit(param.getUnit());
    }

    public String getUdunit() {
      String cu =  Util.cleanUnit(param.getUnit());
         try {
           SimpleUnit su1 = SimpleUnit.factoryWithExceptions(cu);
           return (su1.isUnknownUnit()) ? "UNKNOWN" : su1.toString();
         } catch (Exception e) {
           return "FAIL "+e.getMessage();
         }
    }

    public int getNumber() {
      return param.getNumber();
    }

  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id: DatasetSourceType.java 63 2006-07-12 21:50:51Z edavis $

package thredds.cataloggen.config;

/**
 * Type-safe enumeration of CatalogGen DatasetSource types.
 *
 * @author Ethan Davis (from John Caron's thredds.catalog.ServiceType)
 * @version $Revision: 63 $
 */
public final class DatasetSourceType
{
  private static java.util.HashMap hash = new java.util.HashMap(20);

  public final static DatasetSourceType LOCAL = new DatasetSourceType( "Local");
  public final static DatasetSourceType DODS_FILE_SERVER = new DatasetSourceType( "DodsFileServer");
  public final static DatasetSourceType DODS_DIR = new DatasetSourceType( "DodsDir");
  public final static DatasetSourceType GRADS_DATA_SERVER = new DatasetSourceType( "GrADSDataServer");

  private String typeName;
  private DatasetSourceType( String name)
  {
    this.typeName = name;
    hash.put( name, this);
  }

  /**
   * Find the DatasetSourceType that matches this name.
   * @param name
   * @return DatasetSourceType or null if no match.
   */
  public static DatasetSourceType getType( String name)
  {
    if ( name == null) return null;
    return ( (DatasetSourceType) hash.get( name));
  }

  /**
   * Return the string name.
   */
  public String toString()
  {
    return typeName;
  }

}


package ucar.nc2.ogc.gml;

import net.opengis.gml.x32.CodeWithAuthorityType;
import ucar.nc2.ft.StationTimeSeriesFeature;

/**
 * Created by cwardgar on 2014/02/26.
 */
public abstract class NcCodeWithAuthorityType {
    // wml2:Collection/wml2:observationMember/om:OM_Observation/om:featureOfInterest/wml2:MonitoringPoint/gml:identifier
    public static CodeWithAuthorityType initIdentifier(
            CodeWithAuthorityType identifier, StationTimeSeriesFeature stationFeat) {
        // @codespace
        identifier.setCodeSpace("http://unidata.ucar.edu/");

        // TEXT
        identifier.setStringValue(stationFeat.getName());

        return identifier;
    }

    private NcCodeWithAuthorityType() { }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id: DatasetSource.java 63 2006-07-12 21:50:51Z edavis $

package thredds.cataloggen.config;

import thredds.catalog.*;
import thredds.cataloggen.config.CatalogRefInfo;
import thredds.cataloggen.DatasetEnhancer1;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

// @todo Add state machine to control what can happen when.
// State machine: 1) constructed and modified, 2) expanded, 3) named,
// 4) metadata-ified, and 5) sorted.

// @todo Implement DatasetSource type using Strategy pattern rather than subclassing.
// I.e., extract abstract methods into interface, config (type given to
// DatasetSource.factory()) determines the implementation used.
// Current subclassing is cumbersome and not so flexible as the factory must be
// modified when new implementation is added.

/**
 * Represents a source from which a collection of datasets can be found and
 * expanded into an InvCatalog. Besides expanding the DatasetSource, the datasets
 * in the resulting catalog can be named, sorted, and have metadata added to them.
 *
 * <p>
 * Currently an abstract class, three abstract methods must be implemented to
 * make a concrete subclass. These methods are getTopLevelDataset():InvDataset,
 * isCollection():boolean, and expandThisLevel(InvDataset):List. (In previous
 * versions, expandThisType():void was the only abstract method.)
 * </p>
 *
 * @author Ethan Davis
 * @version $Id: DatasetSource.java 63 2006-07-12 21:50:51Z edavis $
 */
abstract public class DatasetSource
{
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetSource.class);

  /** The catalog that results from expanding this DatasetSource.  */
  protected InvCatalog resultingCatalog;

  /** Top-level dataset of the catalog generated by this DatasetSource. */
  protected InvDataset accessPointDataset;

  /** Information on the catalogRefs in the resulting catalog. */
  protected List catalogRefInfoList = new ArrayList();

  // attributes of the datasetSource element
  private String name;
  protected DatasetSourceType type;
  private DatasetSourceStructure structure;
  private boolean flatten = false;
  private String accessPoint;

  private String prefixUrlPath;

  // If true, create CatalogRefs for all collection datasets, otherwise
  // recurse into all collection datasets (default false).
  protected boolean createCatalogRefs = false;
//  protected boolean expandCatalogRef = false;
// @todo Decide if expandCatalogRef is done here in DatasetSource or by CatalogGen (or other DatasetSource caller).

  // Required child ResultService
  private ResultService resultService;

  // list of optional children of the datasetNamer element
  protected List datasetNamerList = new ArrayList();
  protected List datasetFilterList = new ArrayList();
  protected DatasetSorter datasetSorter = null;

  protected List datasetEnhancerList = new ArrayList();
  protected boolean addDatasetSize;

  // for validation
  protected boolean isValid = true;
  protected StringBuffer msgLog = new StringBuffer();

  /** Default Constructor */
  protected DatasetSource(){}

  /**
   * Factory method for this abstract class.
   * @param name
   * @param type
   * @param structure
   * @param accessPoint
   *
   * @throws IllegalArgumentException if DatasetSource type is null or unsupported.
   */
  public static final DatasetSource newDatasetSource( String name, DatasetSourceType type,
                                             DatasetSourceStructure structure,
                                             String accessPoint,
                                             ResultService resultService )
  {
    if ( type == null)
    {
      String tmpMsg = "DatasetSource type cannot be null";
      logger.error( "newDatasetSource(): " + tmpMsg);
      throw new IllegalArgumentException( tmpMsg);
    }

    DatasetSource tmpDsSource = null;

    if ( type == DatasetSourceType.getType( "Local"))
    {
      tmpDsSource = new LocalDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "DodsDir"))
    {
      tmpDsSource = new DodsDirDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "DodsFileServer"))
    {
      tmpDsSource = new DodsFileServerDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "GrADSDataServer"))
    {
      tmpDsSource = new GrADSDataServerDatasetSource();
    }
    else
    {
      String tmpMsg = "Unsupported DatasetSource type <" + type.toString() + ">.";
      logger.error( "newDatasetSource(): " + tmpMsg);
      throw new IllegalArgumentException( tmpMsg);
    }

    tmpDsSource.setName( name);
    tmpDsSource.setStructure( structure);
    tmpDsSource.setAccessPoint( accessPoint);
    tmpDsSource.setResultService( resultService);
    // Test validity and append messages to log.

    logger.debug( "DatasetSource(): constructor done.");
    StringBuilder log = new StringBuilder();
    if ( tmpDsSource.validate( log))
    {
      logger.debug( "DatasetSource(): new DatasetSource is valid: {}", log.toString());
    }
    else
    {
      logger.debug( "DatasetSource(): new DatasetSource is invalid: {}", log.toString());
    }

    return( tmpDsSource);
  }

  public InvCatalog getResultingCatalog() { return( this.resultingCatalog); }

  public List getCatalogRefInfoList()
  {
    return catalogRefInfoList;
  }

  public String getName() { return( this.name); }
  public void setName( String name) { this.name = name; }

  public DatasetSourceType getType() { return( this.type); }

  public DatasetSourceStructure getStructure() { return( this.structure); }
  public void setStructure( DatasetSourceStructure structure)
  {
    this.structure = structure;
    if ( this.structure == DatasetSourceStructure.FLAT)
      flatten = true;
    else
      flatten = false;
  }

  public boolean isFlatten() { return( this.flatten); }

  public String getAccessPoint() { return( this.accessPoint); }
  public void setAccessPoint( String accessPoint)
  { this.accessPoint = accessPoint; }

  public String getPrefixUrlPath() { return( this.prefixUrlPath); }
  public void setPrefixUrlPath( String prefixUrlPath )
  { this.prefixUrlPath = prefixUrlPath; }

  public ResultService getResultService() { return( this.resultService); }
  public void setResultService( ResultService resultService)
  { this.resultService = resultService; }

  public boolean isCreateCatalogRefs()
  { return( this.createCatalogRefs); }
  public void setCreateCatalogRefs( boolean createCatalogRefs)
  { this.createCatalogRefs = createCatalogRefs; }

//  public boolean isExpandCatalogRef()
//  {
//    return expandCatalogRef;
//  }
//
//  public void setExpandCatalogRef( boolean expandCatalogRef )
//  {
//    this.expandCatalogRef = expandCatalogRef;
//  }

  public java.util.List getDatasetNamerList() { return( this.datasetNamerList); }
  public void addDatasetNamer( DatasetNamer datasetNamer)
  { this.datasetNamerList.add( datasetNamer); }

  public java.util.List getDatasetFilterList() { return( this.datasetFilterList); }
  public void addDatasetFilter( DatasetFilter datasetFilter)
  { this.datasetFilterList.add( datasetFilter); }

  public DatasetSorter getDatasetSorter() { return ( this.datasetSorter ); }
  public void setDatasetSorter( DatasetSorter datasetSorter )
  { this.datasetSorter = datasetSorter; }

  public java.util.List getDatasetEnhancerList() { return( this.datasetEnhancerList ); }
  public void addDatasetEnhancer( DatasetEnhancer1 dsEnhancer)
  {
    if ( this.datasetEnhancerList == null) this.datasetEnhancerList = new ArrayList();
    this.datasetEnhancerList.add( dsEnhancer);
  }

  public boolean isAddDatasetSize() { return( this.addDatasetSize); }
  public void setAddDatasetSize( boolean addDatasetSize )
  { this.addDatasetSize = addDatasetSize; }

  public boolean validate( StringBuilder out)
  {
    this.isValid = true;

    // If log from construction has content, append to validation output msg.
    if (this.msgLog.length() > 0)
    {
      out.append( this.msgLog);
    }

    // Check that name is not null (it can be an empty string).
    if ( this.getName() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (5): null value for name is not valid.");
    }
    // Check that type is not null.
    if ( this.getType() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (6): null value for type is not valid (set with bad string?).");
    }
    // Check that structure is not null.
    if ( this.getStructure() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (7): null value for structure is not valid (set with bad string?).");
    }

    // Validate ResultService child element.
    if ( this.getResultService() != null )
    {
      this.isValid &= this.getResultService().validate( out);
    }

    // Validate DatasetNamer child elements.
    java.util.Iterator dsnIter = this.getDatasetNamerList().iterator();
    while (dsnIter.hasNext())
    {
      this.isValid &= ((DatasetNamer) dsnIter.next()).validate( out);
    }

    // Validate DatasetFilters child elements.
    java.util.Iterator dsfIter = this.getDatasetFilterList().iterator();
    while (dsfIter.hasNext())
    {
      this.isValid &= ((DatasetFilter) dsfIter.next()).validate( out);
    }

    return( this.isValid);
  }

  /** string representation */
  public String toString()
  {
    StringBuffer tmp = new StringBuffer();
    tmp.append( "DatasetSource[name:<" + this.getName() +
                "> type:<" + this.getType() +
                "> structure:<" + this.getStructure() +
                "> accessPoint:<" + this.getAccessPoint() +
                "> and children - " +
                "ResultService(" + this.getResultService().getName() + ") - " +
                "DatasetNamer(" + this.getDatasetNamerList().size() + ") - " +
                "DatasetFilter(" + this.getDatasetFilterList().size() + ")]");
    return( tmp.toString());
  }

  /**
   * Crawl this DatasetSource and generate a new InvCatalog, return the
   * top-level InvDataset.
   *
   * Each object found on the DatasetSource becomes an InvDataset if it is
   * accepted by at least one DatasetFilter. The catalog reflects the
   * heirarchical structure of the DatasetSource. All datasets are named
   * with the location of the object they represent on the dataset source.
   *
   * @return the top-level InvDataset in the generated InvCatalog.
   *
   * @throws IOException if the accessPoint for this DatasetSource is not a container dataset.
   */
  public InvDataset expand() throws IOException
  {
    // Get the new catalog being generated and its top-level dataset.
    this.resultingCatalog = this.createSkeletonCatalog( prefixUrlPath );
    this.accessPointDataset = (InvDataset) this.resultingCatalog.getDatasets().get( 0);

    // IOException thrown by createSkeletonCatalog() so this check should not be necessary.
    if ( ! this.isCollection( this.accessPointDataset) )
    {
      String tmpMsg = "The access point dataset <" + this.accessPointDataset.getName() + "> must be a collection dataset.";
      logger.warn( "expand(): {}", tmpMsg);
      throw new IOException( tmpMsg);
    }

    // Recurse into directory structure and expand.
    expandRecursive( this.accessPointDataset);

    // Finish the catalog.
    ((InvCatalogImpl) this.resultingCatalog).finish();

    // Remove empty collection datasets. @todo HACK - should use filters instead.
    this.recursivelyRemoveEmptyCollectionDatasets( this.accessPointDataset);

    // Return the top-level dataset.
    return( this.accessPointDataset);
  }

  /**
   *
   * @param accessPoints
   * @return the top-level dataset
   * @throws IOException if the main access point for this DatasetSource is not a container dataset or any of the given access points do not exist or are not under the main access point.
   */
  public InvDataset expand( List accessPoints) throws IOException
  {
    // Get the new catalog being generated and its top-level dataset.
    this.resultingCatalog = this.createSkeletonCatalog( prefixUrlPath );
    this.accessPointDataset = (InvDataset) this.resultingCatalog.getDatasets().get( 0);

    for ( Iterator it = accessPoints.iterator(); it.hasNext(); )
    {
      InvDataset curDs = this.createDataset( (String) it.next(), this.prefixUrlPath );
      // Recurse into directory structure and expand.
      expandRecursiveCollection( this.accessPointDataset, curDs);
    }

    // Finish the catalog.
    ((InvCatalogImpl) this.resultingCatalog).finish();

    // Remove empty collection datasets. @todo HACK - should use filters instead.
    this.recursivelyRemoveEmptyCollectionDatasets( this.accessPointDataset);

    // Return the top-level dataset.
    return( this.accessPointDataset);
  }

  private void expandRecursive( InvDataset collectionDataset)
  {
    // Get all datasets at this level.
    List listAllDatasets = this.expandThisLevel( collectionDataset, prefixUrlPath );
    InvDataset curChildDs = null;

    for( Iterator i = listAllDatasets.iterator(); i.hasNext(); )
    {
      curChildDs = (InvDataset) i.next();

      // Filter out unwanted datasets.
      if ( ! DatasetFilter.acceptDatasetByFilterGroup( this.getDatasetFilterList(), curChildDs, this.isCollection( curChildDs)))
        continue;

      // Expand each collection dataset.
      if ( this.isCollection( curChildDs))
      {
        expandRecursiveCollection( collectionDataset, curChildDs );
      }
      // Otherwise, add dataset to collection.
      else
      {
        // If not flattening the directory structure, add current dataset to parent dataset.
        if ( ! this.isFlatten())
        {
          ((InvDatasetImpl) collectionDataset).addDataset( (InvDatasetImpl) curChildDs);
        }
        // Otherwise, add current dataset to top-level dataset.
        else
        {
          ((InvDatasetImpl) this.accessPointDataset).addDataset( (InvDatasetImpl) curChildDs);
        }
      }
    }
  }

  private void expandRecursiveCollection( InvDataset collectionDataset, InvDataset childDs )
  {
    // If not creating a catalogRef, expand the current dataset.
    // @todo Make this determination based on current dataset.
    if ( ! this.createCatalogRefs)
    {
      // If not flattening the directory structure, add current collection dataset to parent dataset.
      if ( ! this.isFlatten())
      {
        ((InvDatasetImpl) collectionDataset).addDataset( (InvDatasetImpl) childDs);
      }
      this.expandRecursive( childDs);
    }
    // Otherwise, add current dataset as a catalogRef.
    else
    {
      // @todo Determine title and docName based on createCatalogRef object
      String title = childDs.getName();
      String docName = (childDs.getName() == null || childDs.getName().equals( ""))
                       ? "catalog.xml"
                       : childDs.getName() + "/catalog.xml";
//      if ( ! this.accessPoint.equals( this.resultService.getAccessPointHeader()))
//      {
//        // @todo HACK, HACK not always Files
//        String ap = new File( this.accessPoint).toURI().toString();
//        String aph = new File( this.resultService.getAccessPointHeader()).toURI().toString();
//        String relPath = ap.substring( aph.length() );
//        docName = relPath + ((relPath.endsWith( "/")) ? "" : "/") + docName;
//      }

      // Add catalogRef to the current dataset.
      InvCatalogRef curCatRef = new InvCatalogRef( (InvDatasetImpl) collectionDataset, title, docName);
      ((InvDatasetImpl) collectionDataset).addDataset( curCatRef);

      // Create a DatasetSource for the catalogRef and add info to the catalogRefInfoList.
      DatasetSource catRefDsSrc = DatasetSource.newDatasetSource( childDs.getName(), this.getType(), this.getStructure(),
                                                                  childDs.getName(), new ResultService( this.getResultService()));
      this.catalogRefInfoList.add( new CatalogRefInfo( title, docName, childDs, catRefDsSrc));
    }
  }

  /**
   * Crawl this DatasetSource and generate a new InvCatalog with all datasets
   * named, sorted, and organized as defined by this DatasetSource, return the
   * newly generated InvCatalog.
   *
   * @return the generated InvCatalog.
   *
   * @throws IOException if DatasetSource does not reference a container dataset.
   */
  public InvCatalog fullExpand() throws IOException
  {
    logger.debug( "fullExpand(): expanding DatasetSource named \"{}\"", this.getName());

    InvDataset topDs = this.expand();
    InvCatalog generatedCat = topDs.getParentCatalog();

    // Add metadata to all datasets.
    for ( Iterator it = this.getDatasetEnhancerList().iterator(); it.hasNext(); )
    {
      DatasetEnhancer1 dsE = (DatasetEnhancer1) it.next();
      dsE.addMetadata( topDs);
    }

    // Name all datasets.
    logger.debug( "fullExpand(): naming the datasets.");
    this.nameDatasets( (InvDatasetImpl) topDs );

    // Sort all datasets
    logger.debug( "fullExpand(): sorting the datasets.");
    this.sortDatasets( topDs);

    // Return the generated catalog
    ((InvCatalogImpl) generatedCat).finish();
    return( generatedCat);

  }

  /**
   * Creates an InvDataset to represent the dataset at the given location
   * on this DatasetSource.
   *
   * @param datasetLocation a String indicating the location of a dataset.
   * @param prefixUrlPath a path name to be prefixed on the urlPath
   * @return an InvDataset for the given dataset location.
   * @throws IOException if the dataset location does not correspond to an actual dataset or is not below the accessPointHeader directory.
   * @throws NullPointerException if the given dataset location is null.
   */
  abstract protected InvDataset createDataset( String datasetLocation, String prefixUrlPath )
          throws IOException;

  /**
   * Return true if the given dataset is a collection dataset, false otherwise.
   *
   * @param dataset - the InvDataset to test for being a collection dataset.
   * @return true if the given dataset is a collection dataset, false otherwise.
   * @throws NullPointerException if the given InvDataset is null.
   * @throws ClassCastException if the given InvDataset is not a LocalInvDataset.
   */
  abstract protected boolean isCollection( InvDataset dataset);

  /**
   * Return a skeleton InvCatalog for this DatasetSource. The skeleton catalog
   * is unnamed with a single top-level dataset representing the DatasetSource
   * accessPoint and a service built from the DatasetSource ResultService.
   *
   * @param prefixUrlPath
   * @return a skeleton InvCatalog for this DatasetSource.
   *
   * @throws IOException if top-level dataset does not exist or is not a collection dataset.
   */
  abstract protected InvCatalog createSkeletonCatalog( String prefixUrlPath ) throws IOException;

  /**
   * Return a list of the InvDatasets contained in the given collection dataset
   * on this DatasetSource.
   *
   * @param collectionDataset - the collection dataset to be expanded.
   * @param prefixUrlPath
   * @return A list of the InvDatasets contained in the given collection dataset.
   * @throws IllegalArgumentException when given dataset is not a collection dataset.
   * @throws NullPointerException if given dataset is null.
   * @throws ClassCastException if the given InvDataset is not a LocalInvDataset.
   */
  abstract protected List expandThisLevel( InvDataset collectionDataset, String prefixUrlPath );

  /**
   * Use the list of dsNamers to name the given list of datasets.
   *
   * @param datasetContainer - a InvDatasetImpl that contains one or
   *  more datasets to be named.
   */
  private void nameDatasets( InvDatasetImpl datasetContainer)
  {
    if ( this.getDatasetNamerList().isEmpty()) return;
    if ( this.isFlatten())
    {
      logger.debug( "nameDatasets(): structure is FLAT calling nameDatasetList()");
      this.nameDatasetList( datasetContainer);
    }
    else
    {
      logger.debug( "nameDatasets(): structure is DIRECTORY_TREE calling" +
              " nameDatasetTree() on each dataset in dataset container");
      InvDatasetImpl curDs = null;
      for ( int j = 0; j < datasetContainer.getDatasets().size(); j++)
      {
        curDs = (InvDatasetImpl) datasetContainer.getDatasets().get( j);
        this.nameDatasetTree( curDs);
      }
    }
    return;
  }

  /** Name the datasets contained in the given dataset.
   *  The given dataset contains a flat list of datasets. */
  private void nameDatasetList( InvDatasetImpl dataset)
  {
    // Create temporary dataset in which to hold named datasets.
    InvDatasetImpl namedDs = new InvDatasetImpl( dataset,
          "nameDatastList() temp dataset", null, null, null);
    // InvDatasetImpl(parentDs, name, dataType, serviceName, urlPath)
    dataset.addDataset( namedDs);

    // Loop through the DatasetNamers
    DatasetNamer curNamer = null;
    for ( int i = 0; i < this.datasetNamerList.size(); i++)
    {
      curNamer = (DatasetNamer) this.datasetNamerList.get( i);
      logger.debug( "nameDatasetList(): trying namer ({})", curNamer.getName());

      // If the current DatasetNamer adds a new level, create a new dataset.
      InvDatasetImpl addLevelDs = null;
      if ( curNamer.getAddLevel())
      {
        addLevelDs = new InvDatasetImpl( null, curNamer.getName(),
                                         null, null, null );
      }

      // Iterate over remaining unnamed datasets.
      InvDatasetImpl curDs = null;
      java.util.Iterator dsIter = dataset.getDatasets().iterator();
      while ( dsIter.hasNext())
      {
        curDs = (InvDatasetImpl) dsIter.next();
        logger.debug( "nameDatasetList(): try namer on this ds ({}-{})", curDs.getName(), curDs.getUrlPath() );

        // Try to name the current dataset.
        if ( curNamer.nameDataset( curDs))
        {
          logger.debug( "nameDatasetList(): ds named ({})", curDs.getName());
          // If adding a level, add named datasets to the added level dataset.
          if ( curNamer.getAddLevel())
          {
            addLevelDs.addDataset( curDs);
          }
          // Otherwise, add the named datasets to namedDs.
          else
          {
            namedDs.addDataset( curDs);
          }

          // Remove the now-named dataset from list of unnamed datasets.
          dsIter.remove();
        }
      } // END - InvDatasetImpl loop

      // If the namer added a level and a dataset was named by this namer, add the
      // new level to the list of named datasets.
      if ( curNamer.getAddLevel())
      {
        if ( addLevelDs.hasNestedDatasets())
        {
          namedDs.addDataset( addLevelDs);
        }
      }

    } // END - DatasetNamer loop
    namedDs.finish();

    // Once all datasets are named (or unnamable with these DatasetNamers),
    // add all the datasets in namedDs back into the given containerDataset.
    if (logger.isDebugEnabled()) {
      logger.debug( "nameDatasetList(): number of unnamed datasets is " + dataset.getDatasets().size() + ".");
      logger.debug( "nameDatasetList(): add named datasets back to container.");
    }
    for ( int i = 0; i < namedDs.getDatasets().size(); i++)
    {
      dataset.addDataset( (InvDatasetImpl) namedDs.getDatasets().get( i));
    }
    dataset.removeDataset( namedDs);

    return;
  }

  /** Name the datasets in the given dataset hierarchy using this
   *  DatasetSource's list of datasetNamers. */
  private void nameDatasetTree( InvDatasetImpl dataset)
  {
    // If dataset does not have a name, try naming it with dsNamers.
    // @todo Rethink naming of directories (look at how DatasetFilter deals with collection vs atomic datasets).
    if ( dataset.getName().equals("") || ! dataset.hasAccess())
    {
      logger.debug( "nameDatasetTree(): naming dataset ({})...", dataset.getUrlPath());
      DatasetNamer dsN = null;
      for ( int i = 0; i < this.datasetNamerList.size(); i++)
      {
        dsN = (DatasetNamer) this.datasetNamerList.get( i);
        if ( dsN.nameDataset( dataset))
        {
          logger.debug( "nameDatasetTree(): ... used namer ({})", dsN.getName());
          break;
        }
      }
    }

    // Try to name any child datasets.
    InvDatasetImpl curDs = null;
    for ( int j = 0; j < dataset.getDatasets().size(); j++)
    {
      curDs = (InvDatasetImpl) dataset.getDatasets().get( j);
      logger.debug( "nameDatasetTree(): recurse to name child dataset ({})", curDs.getUrlPath());
      this.nameDatasetTree( curDs);
    }

    return;
  }

  private void sortDatasets( InvDataset dataset)
  {
    if ( this.getDatasetSorter() == null)
    {
      // Create default dataset Comparator (matches pre-DatasetSorter behaviour).
      DatasetSorter defaultSorter = new DatasetSorter(
              new java.util.Comparator()
              {
                public int compare( Object obj1, Object obj2 )
                {
                  InvDataset ds1 = (InvDataset) obj1;
                  InvDataset ds2 = (InvDataset) obj2;
                  return ( -ds1.getName().compareTo( ds2.getName() ) );
                }
              }
      );
      defaultSorter.sortNestedDatasets( dataset);
    }
    else
    {
      this.getDatasetSorter().sortNestedDatasets( dataset);
    }

    return;
  }

  private void recursivelyRemoveEmptyCollectionDatasets( InvDataset parentDataset)
  {
    InvDataset curDs = null;
    for ( Iterator it = parentDataset.getDatasets().iterator(); it.hasNext(); )
    {
      curDs = (InvDataset) it.next();

      // Do not remove accessible dataset.
      if ( curDs.hasAccess()) continue;

      // Do not remove catalogRef datasets.
      if ( curDs instanceof InvCatalogRef) continue;

      // Recurse into collection datasets children.
      if ( curDs.hasNestedDatasets())
      {
        this.recursivelyRemoveEmptyCollectionDatasets( curDs);
      }
      // Remove any empty collection datasets.
      else
      {
        it.remove();
      }
    }
  }

  public boolean equals( Object o )
  {
    if ( this == o ) return true;
    if ( !( o instanceof DatasetSource ) ) return false;

    final DatasetSource datasetSource = (DatasetSource) o;

    if ( createCatalogRefs != datasetSource.createCatalogRefs ) return false;
//    if ( expandCatalogRef != datasetSource.expandCatalogRef ) return false;
    if ( flatten != datasetSource.flatten ) return false;
    if ( accessPoint != null ? !accessPoint.equals( datasetSource.accessPoint ) : datasetSource.accessPoint != null ) return false;
    if ( datasetFilterList != null ? !datasetFilterList.equals( datasetSource.datasetFilterList ) : datasetSource.datasetFilterList != null ) return false;
    if ( datasetNamerList != null ? !datasetNamerList.equals( datasetSource.datasetNamerList ) : datasetSource.datasetNamerList != null ) return false;
    if ( name != null ? !name.equals( datasetSource.name ) : datasetSource.name != null ) return false;
    if ( resultService != null ? !resultService.equals( datasetSource.resultService ) : datasetSource.resultService != null ) return false;
    if ( type != null ? !type.equals( datasetSource.type ) : datasetSource.type != null ) return false;

    return true;
  }

  public int hashCode()
  {
    if ( hashCode == 0)
    {
      int result = 17;
      result = ( name != null ? name.hashCode() : 0 );
      result = 29 * result + ( type != null ? type.hashCode() : 0 );
      result = 29 * result + ( flatten ? 1 : 0 );
      result = 29 * result + ( accessPoint != null ? accessPoint.hashCode() : 0 );
      result = 29 * result + ( createCatalogRefs ? 1 : 0 );
//      result = 29 * result + ( expandCatalogRef ? 1 : 0 );
      result = 29 * result + ( resultService != null ? resultService.hashCode() : 0 );
      result = 29 * result + ( datasetNamerList != null ? datasetNamerList.hashCode() : 0 );
      result = 29 * result + ( datasetFilterList != null ? datasetFilterList.hashCode() : 0 );
      hashCode = result;
    }
    return hashCode;
  }
  private volatile int hashCode = 0;
}