/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.ma2;

/**
 * Abstraction for matrix operations.
 * A matrix is a rank-2 Array: m[rows, cols].
 * All operations done in double precision
 *
 * @author @caron
 */

public class MAMatrix {
  private Array a;
  private int nrows, ncols;
  private Index ima;

  /**
   * Create an MAMatrix of the given shape.
   * @param nrows number of rows
   * @param ncols number of cols
   */
  public MAMatrix( int nrows, int ncols) {
    this.a = new ArrayDouble.D2(nrows, ncols);
    this.nrows = nrows;
    this.ncols = ncols;
    ima = a.getIndex();
  }
  /**
   * Create an MAMatrix using the given rank-2 array.
   * @param a rank-2 array
   * @exception IllegalArgumentException is a is not rank 2
   */
  public MAMatrix( Array a) {
    this.a = a;
    if (a.getRank() != 2)
      throw new IllegalArgumentException("rank != 2, instead = "+ a.getRank());
    nrows = a.getShape()[0];
    ncols = a.getShape()[1];
    ima = a.getIndex();
  }

  public int getNrows() { return nrows; }
  public int getNcols() { return ncols; }
  public double getDouble(int i, int j) { return a.getDouble(ima.set(i,j)); }
  public void setDouble(int i, int j, double val) { a.setDouble(ima.set(i,j), val); }

  /**
   * Create a new MAMatrix that is the same as this one, with a copy of the backing store.
   */
  public MAMatrix copy() {
    return new MAMatrix( a.copy());
  }

  /**
   * Create a MAMatrix that is the transpose of this one, with the same backing store.
   * Use copy() to get a copy.
   */
  public MAMatrix transpose() {
    return new MAMatrix( a.transpose(0,1));
  }

  /**
   * Get the jth column, return as a MAVector: same backing store.
   */
  public MAVector column(int j) {
    return new MAVector( a.slice(1,j));
  }

  /**
   * Get the ith row, return as a MAVector: same backing store.
   */
  public MAVector row(int i) {
    return new MAVector( a.slice(0,i));
  }

  /**
   * Dot product of matrix and vector: return M dot v
   * @param v dot product with this vector
   * @return MAVector result: new vector
   * @exception IllegalArgumentException if ncols != v.getSize().
   */
  public MAVector dot(MAVector v) {

    if (ncols != v.getNelems())
      throw new IllegalArgumentException("MAMatrix.dot "+ncols+" != "+ v.getNelems());

    ArrayDouble.D1 result = new ArrayDouble.D1(nrows);
    Index imr = result.getIndex();

    for (int i=0; i<nrows; i++) {
      double sum = 0.0;
      for (int k=0; k<ncols; k++)
        sum += getDouble(i, k) * v.getDouble(k);
      result.setDouble( imr.set(i), sum);
    }

    return new MAVector( result);
  }

  /**
   * Matrix multiply: return m1 * m2.
   * @param m1 left matrix
   * @param m2 right matrix
   * @return MAMatrix result: new matrix
   * @exception IllegalArgumentException if m1.getNcols() != m2.getNrows().
   */
  static public MAMatrix multiply(MAMatrix m1, MAMatrix m2) {

    if (m1.getNcols() != m2.getNrows())
      throw new IllegalArgumentException("MAMatrix.multiply "+m1.getNcols()+" != "+ m2.getNrows());
    int kdims = m1.getNcols();

    ArrayDouble.D2 result = new ArrayDouble.D2(m1.getNrows(), m2.getNcols());
    Index imr = result.getIndex();

    for (int i=0; i<m1.getNrows(); i++) {
      for (int j=0; j<m2.getNcols(); j++) {
        double sum = 0.0;
        for (int k=0; k<kdims; k++)
          sum += m1.getDouble(i, k) * m2.getDouble(k, j);
        result.setDouble( imr.set(i,j), sum);
      }
    }

    return new MAMatrix( result);
  }

  /**
   * Matrix multiply by a diagonal matrix, store result in this: this = this * diag
   * @param diag diagonal matrix stored as a Vector
   * @exception IllegalArgumentException if ncols != diag.getNelems().
   */
  public void postMultiplyDiagonal(MAVector diag) {

    if (ncols != diag.getNelems())
      throw new IllegalArgumentException("MAMatrix.postMultiplyDiagonal "+ncols+" != "+ diag.getNelems());

    for (int i=0; i<nrows; i++) {
      for (int j=0; j<ncols; j++) {
        double val = a.getDouble( ima.set(i,j)) * diag.getDouble(j);
        a.setDouble( ima, val);
      }
    }
  }

  /**
   * Matrix multiply by a diagonal matrix, store result in this: this = diag * this
   * @param diag diagonal matrix stored as a Vector
   * @exception IllegalArgumentException if nrows != diag.getNelems().
   */
  public void preMultiplyDiagonal(MAVector diag) {

    if (nrows != diag.getNelems())
      throw new IllegalArgumentException("MAMatrix.preMultiplyDiagonal "+nrows+" != "+ diag.getNelems());

    for (int i=0; i<nrows; i++) {
      for (int j=0; j<ncols; j++) {
        double val = a.getDouble( ima.set(i,j)) * diag.getDouble(i);
        a.setDouble( ima, val);
      }
    }
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib2.table;

import java.util.ArrayList;
import java.util.List;
import ucar.nc2.constants.CDM;
import ucar.nc2.grib.coord.TimeCoordIntvDateValue;
import ucar.nc2.grib.grib2.Grib2Parameter;
import ucar.nc2.grib.grib2.Grib2Pds;
import ucar.nc2.grib.grib2.Grib2Record;
import ucar.nc2.grib.grib2.Grib2Utils;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarPeriod;
import ucar.unidata.util.StringUtil2;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Formatter;

/**
 * Read screen scraped DSS tables. CFSR 093 not needed - screen scraped NCEP ok 2/2/2012
 *
 * @author caron
 * @since 11/3/11
 */
class CfsrLocalTables extends NcepLocalTables {
  CfsrLocalTables(Grib2TableConfig config) {
    super(config);
    initLocalTable();
  }

  // LOOK: Have to override NcepLocalTables. Probably need delegates, not subclasses?
  @Override
  public List<Parameter> getParameters() {
    List<Parameter> result = new ArrayList<>(local.values());
    result.sort(new ParameterSort());
    return result;
  }

  @Override
  public String getTablePath(int discipline, int category, int number) {
    if ((category <= 191) && (number <= 191)) {
      return super.getTablePath(discipline, category, number);
    }
    return config.getPath();
  }

  @Override
  public String getGeneratingProcessName(int genProcess) {
    if (genProcess == 197) {
      return "CFSR";
    }
    return super.getGeneratingProcessName(genProcess);
  }

  // 193
  // Average of N forecasts (or initialized analyses); each product has forecast period of P1 (P1=0 for initialized analyses);
  // products have reference times at intervals of P2, beginning at the given reference time.
  // 194
  // Average of N uninitialized analyses, starting at reference time, at intervals of P2.
  // 195:
  // Average of forecast accumulations. P1 = start of accumulation period. P2 = end of accumulation period.
  // Reference time is the start time of the first forecast, other forecasts at 24-hour intervals.
  // 204
  // Average of forecast accumulations. P1 = start of accumulation period. P2 = end of accumulation period.
  // Reference time is the start time of the first forecast, other forecasts at 6-hour intervals.
  // Number in Ave = number of forecast used
  // 205
  // Average of forecast averages. P1 = start of averaging period. P2 = end of averaging period.
  // Reference time is the start time of the first forecast, other forecasts at 6-hour intervals. Number in Ave = number of forecast used

  @Override
  public int[] getForecastTimeIntervalOffset(Grib2Record gr) {
    Grib2Pds pds = gr.getPDS();
    if (!pds.isTimeInterval()) {
      return null;
    }

    // LOOK this is hack for CFSR monthly combobulation
    // see http://rda.ucar.edu/datasets/ds093.2/#docs/time_ranges.html

    int statType = pds.getOctet(47);
    int n = pds.getInt4StartingAtOctet(50);
    int p2 = pds.getInt4StartingAtOctet(55);
    int p2mp1 = pds.getInt4StartingAtOctet(62);

    int p1 = p2 - p2mp1;
    int start, end;

    switch (statType) {
      case 193:
        start = p1;
        end = p1 + n * p2;
        break;
      case 194:
        start = 0;
        end = n * p2;
        break;
      case 195:
      case 204:
      case 205:
        start = p1;
        end = p2;
        break;
      default:
        throw new IllegalArgumentException("unknown statType " + statType);
    }

    return new int[]{start, end};
  }

  @Override
  public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) {
    Grib2Pds pds = gr.getPDS();
    if (!pds.isTimeInterval()) {
      return null;
    }

    int[] intv = getForecastTimeIntervalOffset(gr);
    assert intv != null;
    int intvLen = intv[1] - intv[0];

    int timeUnitOrg = pds.getTimeUnit();
    int timeUnitConvert = convertTimeUnit(timeUnitOrg);
    CalendarPeriod unitPeriod = Grib2Utils.getCalendarPeriod(timeUnitConvert);
    if (unitPeriod == null) {
      throw new IllegalArgumentException(
          "unknown CalendarPeriod " + timeUnitConvert + " org=" + timeUnitOrg);
    }

    CalendarPeriod.Field fld = unitPeriod.getField();

    CalendarDate start = gr.getReferenceDate().add(intv[0], fld);
    CalendarPeriod period = CalendarPeriod.of(intvLen, fld);

    return new TimeCoordIntvDateValue(start, period);
  }

  /**
   * Only use in GribVariable to decide on variable identity when intvMerge = false. By returning a
   * constant, we dont support intvMerge = false. Problem is we cant reconstruct interval length
   * without reference time, which is not in the pds.
   */
  @Override
  public double getForecastTimeIntervalSizeInHours(Grib2Pds pds) {
    return 6.0;
  }

  private boolean isCfsr2(Grib2Pds pds) {
    int genType = pds.getGenProcessId();
    if ((genType != 82) && (genType != 89)) {
      return false;
    }

    Grib2Pds.PdsInterval pdsIntv = (Grib2Pds.PdsInterval) pds;
    Grib2Pds.TimeInterval[] ti = pdsIntv.getTimeIntervals();
    return ti.length != 1;
  }

  @Override
  public void showSpecialPdsInfo(Grib2Record gr, Formatter f) {
    Grib2Pds pds = gr.getPDS();
    if (!pds.isTimeInterval()) {
      return;
    }
    if (pds.getRawLength() < 65) {
      return;
    }

    /*     Octet(s)	Description
        47	From NCEP Code Table 4.10
        48	Should be ignored
        49	Should be ignored
        50-53	Number of grids used in the average
        54	Should be ignored
        55-58	This is "P2" from the GRIB1 format
        59	From NCEP Code Table 4.10
        60	Should be ignored
        61	Should be ignored
        62-65	This is "P2 minus P1"; P1 and P2 are fields from the GRIB1 format
        66	Should be ignored
        67-70	Should be ignored */

    int statType = pds.getOctet(47);
    int statType2 = pds.getOctet(59);
    int ngrids = pds.getInt4StartingAtOctet(50);
    int p2 = pds.getInt4StartingAtOctet(55);
    int p2mp1 = pds.getInt4StartingAtOctet(62);

    f.format("%nCFSR MM special encoding (NCAR)%n");
    f.format("  (47) Code Table 4.10 = %d%n", statType);
    f.format("  (50-53) N in avg     = %d%n", ngrids);
    f.format("  (55-58) Grib1 P2     = %d%n", p2);
    f.format("  (59) Code Table 4.10 = %d%n", statType2);
    f.format("  (62-65) P2 minus P1  = %d%n", p2mp1);
    f.format("                   P1  = %d%n", p2 - p2mp1);

    int[] intv = getForecastTimeIntervalOffset(gr);
    if (intv == null) {
      return;
    }
    f.format("ForecastTimeIntervalOffset  = (%d,%d)%n", intv[0], intv[1]);
    f.format("      ForecastTimeInterval  = %s%n", getForecastTimeInterval(gr));

    /* Section 4 Octet 58 (possibly 32 bits: 55-58) is the length of the averaging period per unit.
       For cycle fractions, this is 24, for complete monthly averages, it is 6.
       The product of this and the num_in_avg {above} should always equal the total number of hours in a respective month.
     - Section 4 Octet 65 is the hours skipped between each calculation component.
    f.format("%nCFSR MM special encoding (Swank)%n");
    f.format("  (55-58) length of avg period per unit                     = %d%n", p2);
    f.format("  (62-65) hours skipped between each calculation component  = %d%n", p2mp1);
    f.format("  nhours in month %d should be  = %d%n", ngrids * p2, 24 * 31); */
  }

  //////////////////////////////////////////////////////////////////////

  // see http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc.shtml
  private void initLocalTable() {
    String tableName = config.getPath();
    ClassLoader cl = this.getClass().getClassLoader();
    try (InputStream is = cl.getResourceAsStream(tableName)) {
      if (is == null) {
        throw new IllegalStateException("Cant find " + tableName);
      }
      try (BufferedReader br = new BufferedReader(new InputStreamReader(is, CDM.utf8Charset))) {
        while (true) {
          String line = br.readLine();
          if (line == null) {
            break;
          }
          if ((line.length() == 0) || line.startsWith("#")) {
            continue;
          }
          String[] flds = StringUtil2.splitString(line);

          int p1 = Integer.parseInt(flds[0].trim()); // must have a number
          int p2 = Integer.parseInt(flds[1].trim()); // must have a number
          int p3 = Integer.parseInt(flds[2].trim()); // must have a number
          StringBuilder b = new StringBuilder();
          int count = 3;

          while (count < flds.length && !flds[count].equals(".")) {
            b.append(flds[count++]).append(' ');
          }
          String abbrev = b.toString().trim();
          b.setLength(0);
          count++;

          while (count < flds.length && !flds[count].equals(".")) {
            b.append(flds[count++]).append(' ');
          }
          String name = b.toString().trim();
          b.setLength(0);
          count++;

          while (count < flds.length && !flds[count].equals(".")) {
            b.append(flds[count++]).append(' ');
          }
          String unit = b.toString().trim();

          Grib2Parameter s = new Grib2Parameter(p1, p2, p3, name, unit, abbrev, null);
          local.put(makeParamId(p1, p2, p3), s);
        }
      }

    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ui.grid;

import ucar.nc2.ui.gis.GisFeatureRenderer;
import ucar.nc2.ui.widget.FontUtil;
import ucar.unidata.geoloc.*;

import ucar.unidata.util.Format;
import ucar.util.prefs.ui.Debug;

import java.awt.RenderingHints;
import java.util.*;
import java.awt.Color;
import java.awt.Shape;
import java.awt.Font;
import java.awt.geom.AffineTransform;
import java.awt.geom.PathIterator;
import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;

/**
 * Contour rendering.
 *
 * @author caron
 */
public class ContourFeatureRenderer extends GisFeatureRenderer {

  private ProjectionImpl dataProjection;
  private ArrayList contourList = new ArrayList();  // list of ContourFeature-s
  private boolean ShowLabels;

  /**
   * cstr
   */
  public ContourFeatureRenderer(ContourGrid conGrid,
          ProjectionImpl dataProjection) {
    this.dataProjection = dataProjection;
    ShowLabels = true;
    contourList = conGrid.getContourLines();
  }

  /**
   * set switch whether contours labels are desired.
   * default true is set in cstr.
   */
  public void setShowLabels(boolean showlabels) {
    ShowLabels = showlabels;
  }

  public LatLonRect getPreferredArea() {
    return null;
  }

  protected java.util.List getFeatures() {
    // collection of ContourFeature-s
    return contourList;
  }

  protected ProjectionImpl getDataProjection() {
    return dataProjection;
  }

  /**
   * Overrides the GisFeatureRenderer draw() method, to draw contours
   * and with contour labels.
   *
   * @param g                  the Graphics2D context on which to draw
   * @param deviceFromNormalAT transforms "Normalized Device" to Device coordinates
   */
  public void draw(java.awt.Graphics2D g, AffineTransform deviceFromNormalAT) {
    /* OLD WAY
  // make & set desired font for contour label.
  // contour label size in "points" is last arg
  Font font1 = new Font("Helvetica", Font.PLAIN, 25);
  // make a transform to un-flip the font
  AffineTransform unflip = AffineTransform.getScaleInstance(1, -1);
  Font font = font1.deriveFont(unflip);
  g.setFont(font);  */

    /* from original GisFeatureRenderer method draw: */
    g.setColor(Color.black);
    g.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
            RenderingHints.VALUE_ANTIALIAS_OFF);
    g.setStroke(new java.awt.BasicStroke(0.0f));

    Rectangle2D clipRect = (Rectangle2D) g.getClip();
    Iterator siter = getShapes(g, deviceFromNormalAT);

    // draw the contours
    while (siter.hasNext()) {
      Shape s = (Shape) siter.next();
      Rectangle2D shapeBounds = s.getBounds2D();
      if (shapeBounds.intersects(clipRect))
        g.draw(s);
    }

    // additional code beyond GisFeatureRenderer method draw():
    // render contour value for this contour line. */
    if (ShowLabels) {
      Font f = FontUtil.getStandardFont(10).getFont();
      Font saveFont = g.getFont();

      // use world coordinates for position, but draw in "normal" coordinates
      // so that the symbols stay the same size
      AffineTransform deviceFromWorldAT = g.getTransform();
      AffineTransform normalFromWorldAT;
      // transform World to Normal coords:
      //    normalFromWorldAT = deviceFromNormalAT-1 * deviceFromWorldAT
      try {
        normalFromWorldAT = deviceFromNormalAT.createInverse();
        normalFromWorldAT.concatenate(deviceFromWorldAT);
      } catch (java.awt.geom.NoninvertibleTransformException e) {
        System.out.println(" ContourFeatureRenderer: NoninvertibleTransformException on " + deviceFromNormalAT);
        return;
      }
      g.setTransform(deviceFromNormalAT); // so g now wants "normal coords"
      g.setFont(f);

      siter = getShapes(g, deviceFromNormalAT);
      Iterator CViter = contourList.iterator();
      Point2D worldPt = new Point2D.Double();
      Point2D normalPt = new Point2D.Double();
      float [] coords = new float[6];
      while (siter.hasNext()) {
        Shape s = (Shape) siter.next();
        double contValue = ((ContourFeature) CViter.next()).getContourValue();

        // get position xpos,ypos on this contour where to put label
        // in current world coordinates in the current Shape s.
        PathIterator piter = s.getPathIterator(null);
        //int cs, count=-1;  original
        int cs, count = 12;
        while (! piter.isDone()) {
          count++;
          if (count % 25 == 0) {    // for every 25th position on this path
            cs = piter.currentSegment(coords);

            if (cs == PathIterator.SEG_MOVETO || cs == PathIterator.SEG_LINETO) {
              worldPt.setLocation(coords[0], coords[1]);
              normalFromWorldAT.transform(worldPt, normalPt);  // convert to normal
              // render the contour value to the screen
              g.drawString(Format.d(contValue, 4), (int) normalPt.getX(), (int) normalPt.getY());
            }
          }
          piter.next();
        } // while not done
      }  // end while shape.hasNext()

      // restore original transform and font
      g.setTransform(deviceFromWorldAT);
      g.setFont(saveFont);

    }  // end if ShowLabels == true

    if (Debug.isSet("contour/doLabels")) {
      // get iterator to the class member ArrayList of GisFeature-s
      for (Object aContourList : contourList) {
        //ContourFeature cf = iter.next();
        System.out.println(" ContourFeatureRenderer: contour value = "
                + ((ContourFeature) aContourList).getContourValue());
      }
    }
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ui.widget;

import ucar.nc2.ui.util.ScreenUtils;

import java.awt.*;
import java.beans.*;
import javax.swing.*;

/**
 *  Provides common L&F for managing independent windows
 *  Will reset L&F
 *
 *  example: <pre>
     infoWindow = new IndependentWindow("Dataset Information");
     datasetInfoTA = new TextHistoryPane(500, 100, true);
     Container cp = infoWindow.getContentPane();
     cp.add(datasetInfoTA, BorderLayout.CENTER);
     infoWindow.pack();
     infoWindow.setSize(700,700);
     infoWindow.setLocation(100,100);
 *   </pre>
 *
 * @author John Caron
 */
public class IndependentWindow extends JFrame {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(IndependentWindow.class);

  /** constructor
     @param title       Window title
     @param iconImage   image to show when iconified
   */
  public IndependentWindow(String title, Image iconImage) {
    super(title);

    // L&F may change
    UIManager.addPropertyChangeListener(new PropertyChangeListener() {
      public void propertyChange(PropertyChangeEvent e) {
        if (e.getPropertyName().equals("lookAndFeel"))
          SwingUtilities.updateComponentTreeUI(IndependentWindow.this);
      }
    });

    if (null != iconImage)
      setIconImage( iconImage);
  }

  /** constructor
     @param title       Window title.
     @param comp the COmponent to put in the window.
   */
  public IndependentWindow(String title, Image iconImage, Component comp) {
    this(title, iconImage);
    setComponent(comp);
  }

  public void setComponent(Component comp ) {
    Container cp = getContentPane();
    cp.removeAll();
    cp.add(comp, BorderLayout.CENTER);
    try {
      pack();
    } catch (java.lang.IllegalArgumentException e) {
      // Ticket ID: HEM-237554
      // I'm using IceWM window manager under Linux, and it does not support changing the icon on the top left side of the window.
      // This crashes the whole thing. I dont think this should be a fatal exception.
      // It would be helpful for future releases to catch this exception and let the program go ahead without the icon.
      log.error("Possible problem setting icon (?)", e);
    }
  }

  /** show the window. */
  public void show() {
    setState( Frame.NORMAL );   // deiconify if needed
    super.toFront();
    // need to put on event thread
    SwingUtilities.invokeLater(new Runnable() {
      public void run() {
        IndependentWindow.super.show();
      }
    });
  }

  /** show if not iconified */
  public void showIfNotIconified() {
    if (getState() == Frame.ICONIFIED) return;
    // need to put on event thread
    SwingUtilities.invokeLater(new Runnable() {
      public void run() {
        IndependentWindow.super.show();
      }
    });
  }

  @Override
  public void setBounds(Rectangle r) {
    // keep window on the screen
    Rectangle screenSize = ScreenUtils.getScreenVirtualSize();
    Rectangle result = r.intersection(screenSize);
    if (!result.isEmpty())
      super.setBounds(result);
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.iosp.gini;

import ucar.nc2.constants.DataFormatType;
import ucar.ma2.*;
import ucar.nc2.Variable;
import ucar.nc2.iosp.AbstractIOServiceProvider;

import java.io.*;
import java.awt.image.*;
import java.util.List;
import java.util.zip.Inflater;
import java.util.zip.DataFormatException;

/**
 * IOServiceProvider for GINI files.
 */

public class Giniiosp extends AbstractIOServiceProvider {
  final static int Z_DEFLATED = 8;
  final static int DEF_WBITS = 15;

  public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) {
    return Giniheader.isValidFile(raf);
  }

  /////////////////////////////////////////////////////////////////////////////
  // reading

  public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile,
                   ucar.nc2.util.CancelTask cancelTask) throws IOException {

    super.open(raf, ncfile, cancelTask);

    Giniheader headerParser = new Giniheader();
    headerParser.read(raf, ncfile);

    ncfile.finish();
  }

  public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException {
    // subset
    List<Range> ranges = Section.fill(section, v2.getShape()).getRanges();
    Giniheader.Vinfo vinfo = (Giniheader.Vinfo) v2.getSPobject();
    int[] levels = vinfo.levels;

    if (vinfo.compression == 0)
      return readData(v2, vinfo.begin, ranges, levels);
    else if (vinfo.compression == 2)
      return readCompressedData(v2, vinfo.begin, ranges, levels);
    else if (vinfo.compression == 1)
      return readCompressedZlib(v2, vinfo.begin, vinfo.nx, vinfo.ny, ranges, levels);
    else
      return null;
  }

  private float[] handleLevels(byte[] data,  int[] levels) {
    int level = levels[0];
    float[] a = new float[level];
    float[] b = new float[level];
    float[] fdata = new float[data.length];
    int scale = 1;

    for (int i = 0; i < level; i++) {
      int numer = levels[1 + 5 * i] - levels[2 + 5 * i];
      int denom = levels[3 + 5 * i] - levels[4 + 5 * i];
      a[i] = (numer * 1.f) / (1.f * denom);
      b[i] = levels[1 + 5 * i] - a[i] * levels[3 + 5 * i];
    }

    for (int i = 0; i < data.length; i++) {
      int ival = DataType.unsignedByteToShort(data[i]);
      int k = -1;
      for (int j = 0; j < level; j++) {
        if (levels[3 + (j * 5)] <= ival && ival <= levels[4 + (j * 5)]) {
          k = j;
          scale = levels[5 + j * 5];
        }
      }

      if (k >= 0)
        fdata[i] = (a[k] * ival + b[k]) / scale;
      else
        fdata[i] = 0;

    }
    return fdata;
  }

  private Array makeArray(byte[] data, int[] levels, int[] shape)
  {
    // Default (if no level data) is to just return an array from the bytes.
    Object store = data;
    DataType dt = DataType.BYTE;

    // If have levels, convert data to float and set-up to use that for array
    if (levels != null) {
      store = handleLevels(data, levels);
      dt = DataType.FLOAT;
    }

    // Create array and return
    return Array.factory(dt, shape, store);
  }

  // all the work is here, so can be called recursively
  private Array readData(ucar.nc2.Variable v2, long dataPos, List<Range> ranges,
                         int[] levels) throws IOException, InvalidRangeException {
    // Get to the proper offset and read in the data
    raf.seek(dataPos);
    int data_size = (int) (raf.length() - dataPos);
    byte[] data = new byte[data_size];
    raf.readFully(data);

    // Turn it into an array
    Array array = makeArray(data, levels, v2.getShape());
    return array.sectionNoReduce(ranges);
  }

  // for the compressed data read all out into a array and then parse into requested
  private Array readCompressedData(ucar.nc2.Variable v2, long dataPos, List<Range> ranges,
                                   int[] levels) throws IOException, InvalidRangeException {
    // Get to the proper offset and read in the rest of the compressed data
    raf.seek(dataPos);
    int data_size = (int) (raf.length() - dataPos);
    byte[] data = new byte[data_size];
    raf.readFully(data);

    // Send the compressed data to ImageIO (to handle PNG)
    ByteArrayInputStream ios = new ByteArrayInputStream(data);
    BufferedImage image = javax.imageio.ImageIO.read(ios); // LOOK why ImageIO ??
    DataBuffer db = image.getData().getDataBuffer();

    // If the image had byte data, turn into an array
    if (db instanceof DataBufferByte) {
      DataBufferByte dbb = (DataBufferByte) db;
      Array array = makeArray(dbb.getData(), levels, v2.getShape());
      if (levels == null)
        v2.setCachedData(array, false);
      return array.sectionNoReduce(ranges);
    }

    return null;
  }

  private Array readCompressedZlib(ucar.nc2.Variable v2, long dataPos, int nx, int ny,
                                   List<Range> ranges, int[] levels) throws IOException, InvalidRangeException {
    // Get to the proper offset and read in the rest of the compressed data
    raf.seek(dataPos);
    int data_size = (int) (raf.length() - dataPos);     //  or 5120 as read buffer size
    byte[] data = new byte[data_size];
    raf.readFully(data);

    // Buffer for decompressing data
    byte[] uncomp = new byte[nx * ny];
    int offset = 0;

    // Set-up zlib decompression (inflation)
    Inflater inflater = new Inflater(false);
    inflater.setInput(data);

    // Loop while the inflater has data and we have space in final buffer
    // This will end up ignoring the last few compressed bytes, which
    // correspond to the end of file marker, which is a single row of pixels
    // of alternating 0/255.
    while (inflater.getRemaining() > 0 && offset < uncomp.length) {
      // Try to decompress what's left, which ends up decompressing one block
      try {
        offset += inflater.inflate(uncomp, offset, uncomp.length - offset);
      } catch (DataFormatException ex) {
        System.out.println("ERROR on inflation " + ex.getMessage());
        ex.printStackTrace();
        throw new IOException(ex.getMessage());
      }

      // If the last block finished...
      if (inflater.finished()) {
        // See if anything's left
        int bytesLeft = inflater.getRemaining();
        if (bytesLeft > 0) {
          // Figure out where we are in the input data
          int inputOffset = data_size - bytesLeft;

          // Check if remaining data are zlib--if not copy out and bail
          byte[] b2 = new byte[2];
          System.arraycopy(data, inputOffset, b2, 0, b2.length);
          if (!isZlibHed(b2)) {
            System.arraycopy(data, inputOffset, uncomp, offset, bytesLeft);
            break;
          }

          // Otherwise, set up for decompressing next block
          inflater.reset();
          inflater.setInput(data, inputOffset, bytesLeft);
        }
      }
    }
    inflater.end();

    // Turn the decompressed data into an array, caching as appropriate
    Array array = makeArray(uncomp, levels, v2.getShape());
    if (levels == null && array.getSize() < Variable.defaultSizeToCache)
      v2.setCachedData(array, false);
    return array.sectionNoReduce(ranges);
  }

  static boolean isZlibHed(byte[] buf) {
    short b0 = DataType.unsignedByteToShort(buf[0]);
    short b1 = DataType.unsignedByteToShort(buf[1]);

    if ((b0 & 0xf) == Z_DEFLATED) {
      if ((b0 >> 4) + 8 <= DEF_WBITS) {
        if ((((b0 << 8) + b1) % 31) == 0) {
          return true;
        }
      }
    }

    return false;
  }

  public String getFileTypeId() {
    return DataFormatType.GINI.getDescription();
  }

  public String getFileTypeDescription() {
    return "GOES Ingest and NOAAPORT Interface";
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.dt.grid;

import org.w3c.dom.Attr;

import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;
import ucar.nc2.*;
import ucar.nc2.constants.*;
import ucar.nc2.dataset.*;
import ucar.nc2.dataset.transform.AbstractTransformBuilder;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarDateRange;
import ucar.unidata.geoloc.*;
import ucar.unidata.geoloc.projection.LatLonProjection;
import ucar.unidata.util.Parameter;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Write CF compliant gridded data
 * version 2
 *
 * @author caron
 * @since 6/18/2014
 */
public class CFGridWriter2 {
  static private final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(CFGridWriter2.class);

  /**
   * Compute the size of the file without writing
   *
   * @param gds              the GridDataset
   * @param gridList         the list of variables to be written, or null for all
   * @param llbb             the lat/lon bounding box, or null for all
   * @param projRect         the projection bounding box, or null for all
   * @param horizStride      the x and y stride
   * @param zRange           the z stride
   * @param dateRange        date range, or null for all
   * @param stride_time      the time stride
   * @param addLatLon        add 2D lat/lon coordinates if needed
   * @return total bytes written
   * @throws IOException
   * @throws InvalidRangeException
   */
  static public long makeSizeEstimate(ucar.nc2.dt.GridDataset gds, List<String> gridList,
                                       LatLonRect llbb, ProjectionRect projRect, int horizStride, Range zRange,
                                       CalendarDateRange dateRange, int stride_time, boolean addLatLon) throws IOException, InvalidRangeException {

    CFGridWriter2 writer2 = new CFGridWriter2();
    return writer2.writeOrTestSize(gds, gridList, llbb, projRect, horizStride, zRange, dateRange, stride_time, addLatLon, true, null);
  }

  /**
   * Write a netcdf/CF file from a GridDataset
   *
   * @param gds              the GridDataset
   * @param gridList         the list of variables to be written, or null for all
   * @param llbb             the lat/lon bounding box, or null for all
   * @param projRect         the projection bounding box, or null for all
   * @param horizStride      the x and y stride
   * @param zRange           the z stride
   * @param dateRange        date range, or null for all
   * @param stride_time      the time stride
   * @param addLatLon        add 2D lat/lon coordinates if needed
   * @param writer           this does the actual writing
   * @return total bytes written
   * @throws IOException
   * @throws InvalidRangeException
   */
  static public long writeFile(ucar.nc2.dt.GridDataset gds, List<String> gridList,
                                       LatLonRect llbb, ProjectionRect projRect, int horizStride, Range zRange,
                                       CalendarDateRange dateRange, int stride_time, boolean addLatLon, NetcdfFileWriter writer) throws IOException, InvalidRangeException {

    CFGridWriter2 writer2 = new CFGridWriter2();
    return writer2.writeOrTestSize(gds, gridList, llbb, projRect, horizStride, zRange, dateRange, stride_time, addLatLon, false, writer);
  }

  /**
   *
   * @param gds              the GridDataset
   * @param gridList         the list of variables to be written, or null for all
   * @param llbb             the lat/lon bounding box, or null for all
   * @param projRect         the projection bounding box, or null for all
   * @param horizStride      the x and y stride
   * @param zRange           the z stride
   * @param dateRange        date range, or null for all
   * @param stride_time      the time stride
   * @param addLatLon        add 2D lat/lon coordinates if needed
   * @param testSizeOnly     dont write, just return size
   * @param writer           this does the actual writing
   * @return total bytes written
   * @throws IOException
   * @throws InvalidRangeException
   */
  private long writeOrTestSize(ucar.nc2.dt.GridDataset gds, List<String> gridList,
                              LatLonRect llbb, ProjectionRect projRect, int horizStride, Range zRange,
                              CalendarDateRange dateRange, int stride_time,
                              boolean addLatLon, boolean testSizeOnly,
                              NetcdfFileWriter writer) throws IOException, InvalidRangeException {

    NetcdfDataset ncd = (NetcdfDataset) gds.getNetcdfFile();

    List<Variable> varList = new ArrayList<>();         // could make these Sets
    List<String> varNameList = new ArrayList<>();
    List<CoordinateAxis> axisList = new ArrayList<>();

    if (gridList == null) {   // want all of them
      gridList = new ArrayList<>();
      for (GridDatatype grid : gds.getGrids())
        gridList.add(grid.getName());
    }

    LatLonRect resultBB = null;
    long total_size = 0;

    // add each desired Grid to the new file   LOOK Might be better to group by coordSys
    for (String gridName : gridList) {
      if (varNameList.contains(gridName))  // already contains
        continue;
      GridDatatype gridOrg = gds.findGridDatatype(gridName);
      if (gridOrg == null) {
        log.debug("writeOrTestSize cant find grid %s - skipping%n", gridName);
        continue;
      }

      GridCoordSystem gcsOrg = gridOrg.getCoordinateSystem();
      // boolean isGlobal = gcsOrg.isGlobalLon();
      varNameList.add(gridName);

      // make subset as needed
      Range timeRange = makeTimeRange(dateRange, gcsOrg.getTimeAxis1D(), stride_time);
      Range zRangeUse = makeVerticalRange(zRange, gcsOrg.getVerticalAxis());             // LOOK no vert stride

      GridDatatype gridWant;
      List<Range> yxRanges = new ArrayList<>(2);
      if (projRect != null) {
        makeHorizRange(gcsOrg, projRect, horizStride, yxRanges);
        gridWant = gridOrg.makeSubset(null, null, timeRange, zRangeUse, yxRanges.get(0), yxRanges.get(1));

      } else {

        if (llbb == null) {
          yxRanges.add(null);
          yxRanges.add(null);
        } else {
          yxRanges = gcsOrg.getRangesFromLatLonRect(llbb);
        }

        if ((null != timeRange) || (zRangeUse != null) || (llbb != null) || (horizStride > 1)) {
          gridWant = gridOrg.makeSubset(timeRange, zRangeUse, llbb, 1, horizStride, horizStride);
        } else
          gridWant = gridOrg;
      }

      GridCoordSystem gcsWant = gridWant.getCoordinateSystem();
      LatLonRect gridBB = gcsWant.getLatLonBoundingBox();
      if (resultBB == null)
        resultBB = gridBB;
      else
        resultBB.extend(gridBB);

      Variable gridV = gridWant.getVariable(); // LOOK  WTF ??
      varList.add(gridV);
      total_size += gridV.getSize() * gridV.getElementSize();

      // add coordinate axes
      addCoordinateAxis(gcsWant, varNameList, varList, axisList);

      // add coordinate transform variables
      addCoordinateTransform(gcsWant, ncd, varNameList, varList);

      //Add Variables from the formula_terms
      total_size += processTransformationVars(varList, varNameList, ncd, gds, gridWant, timeRange, zRangeUse, yxRanges.get(0), yxRanges.get(1), horizStride, horizStride);

      // optional lat/lon
      if (addLatLon) {
        Projection proj = gcsWant.getProjection();
        if ((null != proj) && !(proj instanceof LatLonProjection)) {
          total_size += addLatLon2D(ncd, varList, proj, gcsWant.getXHorizAxis(), gcsWant.getYHorizAxis());
          addLatLon = false; // ??
        }
      }
    }

    if (testSizeOnly)
      return total_size;

    ////////////////////////////////////////////////////////////////////
    // start writing here

    // check size is ok
    boolean isLargeFile = isLargeFile(total_size);
    writer.setLargeFile(isLargeFile);
    if (resultBB != null)
      addGlobalAttributes(writer, gds, resultBB);

    // use fileWriter to copy the variables
    FileWriter2 fileWriter = new FileWriter2(writer);
    for (Variable v : varList)
      fileWriter.addVariable(v);

    addCFAnnotations(writer, gds, gridList, ncd, axisList, addLatLon);

    writer.create();

    // use fileWriter to copy the data
    fileWriter.copyVarData(varList, null, null);
    //updateGeospatialRanges(writer, llrect );
    writer.close();

    // this writes the data to the new file.
    return total_size; // ok
  }

  private boolean isLargeFile(long total_size) {
    boolean isLargeFile = false;
    long maxSize = 2 * 1000 * 1000 * 1000;  // LOOK why not use exact
    if (total_size > maxSize) {
      log.debug("Request size = {} Mbytes", total_size / 1000 / 1000);
      isLargeFile = true;
    }
    return isLargeFile;
  }

  private void makeHorizRange(GridCoordSystem gcsOrg, ProjectionRect projRect, int horizStride, List<Range> yxRanges) throws InvalidRangeException {
    if (gcsOrg.getXHorizAxis().getRank() > 1 || gcsOrg.getYHorizAxis().getRank() > 1) {
      throw new IllegalArgumentException("Coordinate systems with 2D horizontal axis are not supported");
    }

    CoordinateAxis1D xAxis = (CoordinateAxis1D) gcsOrg.getXHorizAxis();
    double[] xCoords = xAxis.getCoordValues();

    CoordinateAxis1D yAxis = (CoordinateAxis1D) gcsOrg.getYHorizAxis();
    double[] yCoords = yAxis.getCoordValues();

    ProjectionRect fullBB = new ProjectionRect(xCoords[0], yCoords[0], xCoords[xCoords.length - 1], yCoords[yCoords.length - 1]);

    if (!projRect.intersects(fullBB)) {
      throw new InvalidRangeException("BBOX must intersect grid BBOX, minx=" + xCoords[0] + ", miny=" + yCoords[0] + ", maxx=" + xCoords[xCoords.length - 1] + ", maxy=" + yCoords[yCoords.length - 1]);
    }

    ProjectionRect.intersect(fullBB, projRect, projRect);

    ProjectionPoint lowerLeft = projRect.getLowerLeftPoint();
    ProjectionPoint upperRigth = projRect.getUpperRightPoint();
    double minX = lowerLeft.getX();
    double minY = lowerLeft.getY();
    double maxX = upperRigth.getX();
    double maxY = upperRigth.getY();

    int minY_idx = yAxis.findCoordElement(minY);
    int maxY_idx = yAxis.findCoordElement(maxY);

    // When yAxis is increasing, "minY_idx < maxY_idx == true".
    // When yAxis is decreasing, "minY_idx > maxY_idx == true".
    // To handle both cases, we need to use the min() and max() functions.
    Range yRange = new Range(Math.min(minY_idx, maxY_idx), Math.max(minY_idx, maxY_idx), horizStride);

    int minX_idx = xAxis.findCoordElement(minX);
    int maxX_idx = xAxis.findCoordElement(maxX);
    Range xRange = new Range(Math.min(minX_idx, maxX_idx), Math.max(minX_idx, maxX_idx), horizStride);

    yxRanges.add(0, yRange);
    yxRanges.add(1, xRange);
  }

  private void addGlobalAttributes(NetcdfFileWriter writer, ucar.nc2.dt.GridDataset gds, LatLonRect llbb) {
    // global attributes
    for (Attribute att : gds.getGlobalAttributes()) {
      if (att.getShortName().equals(CDM.FILE_FORMAT)) continue;
      if (att.getShortName().equals(_Coordinate._CoordSysBuilder)) continue;
      writer.addGroupAttribute(null, att);
    }

    Attribute att = gds.findGlobalAttributeIgnoreCase(CDM.CONVENTIONS);
    if (att == null || !att.getStringValue().startsWith("CF-"))  // preserve previous version of CF Convention if it exists
      writer.addGroupAttribute(null, new Attribute(CDM.CONVENTIONS, "CF-1.0"));

    writer.addGroupAttribute(null, new Attribute("History",
            "Translated to CF-1.0 Conventions by Netcdf-Java CDM (CFGridWriter2)\n" +
                    "Original Dataset = " + gds.getLocation() + "; Translation Date = " + CalendarDate.present()));

    // this will replace any existing
    writer.addGroupAttribute(null, new Attribute(ACDD.LAT_MIN, llbb.getLatMin()));
    writer.addGroupAttribute(null, new Attribute(ACDD.LAT_MAX, llbb.getLatMax()));
    writer.addGroupAttribute(null, new Attribute(ACDD.LON_MIN, llbb.getLonMin()));
    writer.addGroupAttribute(null, new Attribute(ACDD.LON_MAX, llbb.getLonMax()));
  }

  private void addCFAnnotations(NetcdfFileWriter writer, ucar.nc2.dt.GridDataset gds, List<String> gridList, NetcdfDataset ncd,
                                List<CoordinateAxis> axisList, boolean addLatLon) {

    //Group root = ncfile.getRootGroup();
    for (String gridName : gridList) {
      GridDatatype grid = gds.findGridDatatype(gridName);
      Variable newV = writer.findVariable(gridName);
      if (newV == null) {
        log.debug("NetcdfCFWriter cant find " + gridName + " in gds " + gds.getLocation());
        continue;
      }

      // annotate Variable for CF
      StringBuilder sbuff = new StringBuilder();
      GridCoordSystem gcs = grid.getCoordinateSystem();
      for (Variable axis : gcs.getCoordinateAxes()) {
        sbuff.append(axis.getFullName()).append(" ");
      }
      if (addLatLon)
        sbuff.append("lat lon");
      newV.addAttribute(new Attribute(CF.COORDINATES, sbuff.toString()));

      // looking for coordinate transform variables
      for (CoordinateTransform ct : gcs.getCoordinateTransforms()) {
        Variable v = ncd.findVariable(ct.getName());
        if (ct.getTransformType() == TransformType.Projection)
          newV.addAttribute(new Attribute(CF.GRID_MAPPING, v.getFullName()));
      }
    }

    for (CoordinateAxis axis : axisList) {
      Variable newV = writer.findVariable(axis.getFullNameEscaped());
      if ((axis.getAxisType() == AxisType.Height) || (axis.getAxisType() == AxisType.Pressure) || (axis.getAxisType() == AxisType.GeoZ)) {
        if (null != axis.getPositive())
          newV.addAttribute(new Attribute(CF.POSITIVE, axis.getPositive()));
      }
      if (axis.getAxisType() == AxisType.Lat) {
        newV.addAttribute(new Attribute(CDM.UNITS, "degrees_north"));
        newV.addAttribute(new Attribute(CF.STANDARD_NAME, "latitude"));
      }
      if (axis.getAxisType() == AxisType.Lon) {
        newV.addAttribute(new Attribute(CDM.UNITS, "degrees_east"));
        newV.addAttribute(new Attribute(CF.STANDARD_NAME, "longitude"));
      }
      if (axis.getAxisType() == AxisType.GeoX) {
        newV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_x_coordinate"));
      }
      if (axis.getAxisType() == AxisType.GeoY) {
        newV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_y_coordinate"));
      }
    }

    // coordinate transform variables : must convert false easting, northing to km
    List<Variable> ctvList = new ArrayList<>();
    for (ucar.nc2.dt.GridDataset.Gridset gridSet : gds.getGridsets()) {
      ucar.nc2.dt.GridCoordSystem gcs = gridSet.getGeoCoordSystem();
      ProjectionCT pct = gcs.getProjectionCT();
      if (pct != null) {
        Variable v = writer.findVariable(pct.getName()); // look for the ctv
        if ((v != null) && !ctvList.contains(v)) {
          convertProjectionCTV((NetcdfDataset) gds.getNetcdfFile(), v);
          ctvList.add(v);
        }
      }
    }
  }

  private void convertProjectionCTV(NetcdfDataset ds, Variable ctv) {
    Attribute att = ctv.findAttribute(_Coordinate.TransformType);
    if ((null != att) && att.getStringValue().equals("Projection")) {
      Attribute east = ctv.findAttribute("false_easting");
      Attribute north = ctv.findAttribute("false_northing");
      if ((null != east) || (null != north)) {
        double scalef = AbstractTransformBuilder.getFalseEastingScaleFactor(ds, ctv);
        if (scalef != 1.0) {
          convertAttribute(ctv, east, scalef);
          convertAttribute(ctv, north, scalef);
        }
      }
    }
  }

  private void convertAttribute(Variable ctv, Attribute att, double scalef) {
    if (att == null) return;
    double val = scalef * att.getNumericValue().doubleValue();
    ctv.addAttribute(new Attribute(att.getShortName(), val));
  }

  private long addLatLon2D(NetcdfFile ncfile, List<Variable> varList, Projection proj,
                           CoordinateAxis xaxis, CoordinateAxis yaxis) throws IOException {

    double[] xData = (double[]) xaxis.read().get1DJavaArray(double.class);
    double[] yData = (double[]) yaxis.read().get1DJavaArray(double.class);

    List<Dimension> dims = new ArrayList<>();
    dims.add(yaxis.getDimension(0));
    dims.add(xaxis.getDimension(0));

    Variable latVar = new Variable(ncfile, null, null, "lat");
    latVar.setDataType(DataType.DOUBLE);
    latVar.setDimensions(dims);
    latVar.addAttribute(new Attribute(CDM.UNITS, CDM.LAT_UNITS));
    latVar.addAttribute(new Attribute(CDM.LONG_NAME, "latitude coordinate"));
    latVar.addAttribute(new Attribute(CF.STANDARD_NAME, "latitude"));
    latVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));

    Variable lonVar = new Variable(ncfile, null, null, "lon");
    lonVar.setDataType(DataType.DOUBLE);
    lonVar.setDimensions(dims);
    lonVar.addAttribute(new Attribute(CDM.UNITS, CDM.LON_UNITS));
    lonVar.addAttribute(new Attribute(CDM.LONG_NAME, "longitude coordinate"));
    lonVar.addAttribute(new Attribute(CF.STANDARD_NAME, "longitude"));
    lonVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));

    int nx = xData.length;
    int ny = yData.length;

    // create the data
    ProjectionPointImpl projPoint = new ProjectionPointImpl();
    LatLonPointImpl latlonPoint = new LatLonPointImpl();
    double[] latData = new double[nx * ny];
    double[] lonData = new double[nx * ny];
    for (int i = 0; i < ny; i++) {
      for (int j = 0; j < nx; j++) {
        projPoint.setLocation(xData[j], yData[i]);
        proj.projToLatLon(projPoint, latlonPoint);
        latData[i * nx + j] = latlonPoint.getLatitude();
        lonData[i * nx + j] = latlonPoint.getLongitude();
      }
    }
    Array latDataArray = Array.factory(DataType.DOUBLE, new int[]{ny, nx}, latData);
    latVar.setCachedData(latDataArray, false);

    Array lonDataArray = Array.factory(DataType.DOUBLE, new int[]{ny, nx}, lonData);
    lonVar.setCachedData(lonDataArray, false);

    varList.add(latVar);
    varList.add(lonVar);

    long size = 0;
    size += latVar.getSize() * latVar.getElementSize();
    size += lonVar.getSize() * lonVar.getElementSize();
    return size;
  }

  private Range makeVerticalRange(Range zRange, CoordinateAxis1D vertAxis) throws InvalidRangeException {
    return (zRange != null) && (vertAxis != null) && (vertAxis.getSize() > 1) ? zRange : null;
  }

  private Range makeTimeRange(CalendarDateRange dateRange, CoordinateAxis1DTime timeAxis, int stride_time) throws InvalidRangeException {

    Range timeRange = null;
    if ((dateRange != null) && (timeAxis != null)) {
      int startIndex = timeAxis.findTimeIndexFromCalendarDate(dateRange.getStart());
      int endIndex = timeAxis.findTimeIndexFromCalendarDate(dateRange.getEnd());
      if (startIndex < 0)
        throw new InvalidRangeException("start time=" + dateRange.getStart() + " must be >= " + timeAxis.getCalendarDate(0));
      if (endIndex < 0)
        throw new InvalidRangeException("end time=" + dateRange.getEnd() + " must be >= " + timeAxis.getCalendarDate(0));
      if (stride_time <= 1) stride_time = 1;
      timeRange = new Range(startIndex, endIndex, stride_time);
    }

    return timeRange;

  }

  private void addCoordinateAxis(GridCoordSystem gcs, List<String> varNameList, List<Variable> varList, List<CoordinateAxis> axisList) {

    for (CoordinateAxis axis : gcs.getCoordinateAxes()) {
      if (!varNameList.contains(axis.getFullName())) {
        varNameList.add(axis.getFullName());
        varList.add(axis);                    // LOOK axis hasnt been subset yet !!
        axisList.add(axis);
        //if (timeAxis != null && timeAxis.isInterval()) {
        // LOOK gotta add the bounds  !!!
        //}
      }
    }

  }

  private void addCoordinateTransform(GridCoordSystem gcs, NetcdfFile ncd, List<String> varNameList, List<Variable> varList) {

    List<String> coordTransformNames = new ArrayList<>();

    // look for variables detected as coordinate transforms
    for (CoordinateTransform ct : gcs.getCoordinateTransforms()) {
      coordTransformNames.add(ct.getName());
    }

    // At the very least, LatLon_Projection will not show up in the coordinate transforms,
    // so inspect the variables in varNameList for a grid_mapping attribute, and make sure it
    // gets picked up as well
    // see https://github.com/Unidata/python-workshop/issues/372
    for (String varName : varNameList) {
      Variable v = ncd.findVariable(varName);
      Attribute gridMapping = v.findAttributeIgnoreCase(CF.GRID_MAPPING);
      if (gridMapping != null) {
        String gridMappingName = gridMapping.getStringValue();
        if ((gridMappingName != null) && (!coordTransformNames.contains(gridMappingName))) {
          coordTransformNames.add(gridMappingName);
        }
      }
    }

    // add the coordinate transforms that we found
    for (String ctVarName : coordTransformNames) {
      Variable v = ncd.findVariable(ctVarName);
      if (!varNameList.contains(ctVarName) && (null != v)) {
        varNameList.add(ctVarName);
        varList.add(v);
      }
    }
  }

  /*
   * Process the coordinate transformations (formula_terms) and adds the variables needed for performing that transformation.
   * Subsets the grids variables, if needed.
   * Return size of variables added
   */
  private long processTransformationVars(List<Variable> varList, List<String> varNameList, NetcdfDataset ncd, ucar.nc2.dt.GridDataset gds,
                                         GridDatatype grid, Range timeRange, Range zRangeUse, Range yRange, Range xRange, int y_stride, int x_stride) throws InvalidRangeException {

    long varsSize = 0L;
    List<CoordinateTransform> cctt = grid.getCoordinateSystem().getCoordinateTransforms();
    for (CoordinateTransform ct : cctt) {
      Parameter param = ct.findParameterIgnoreCase(CF.FORMULA_TERMS);

      if (param != null) {
        String[] varStrings = param.getStringValue().split(" ");
        for (int i = 1; i < varStrings.length; i += 2) {
          Variable paramVar = ncd.findVariable(varStrings[i].trim());

          if (!varNameList.contains(varStrings[i]) && (null != paramVar)) {

            if (gds.findGridDatatype(paramVar.getFullName()) != null) {
              //Subset if needed
              if ((null != timeRange) || (zRangeUse != null) || (x_stride > 1 && y_stride > 1) || (yRange != null || xRange != null)) {
                GridDatatype complementaryGrid = gds.findGridDatatype(paramVar.getFullName());
                complementaryGrid = complementaryGrid.makeSubset(null, null, timeRange, zRangeUse, yRange, xRange);
                paramVar = complementaryGrid.getVariable();
              }
            } else {
              //Also have to subset the var if it is not a grid but has vertical dimension (the dimensionless vars in the formula) and zRangeUse != null
              if (zRangeUse != null && paramVar.getRank() == 1) {
                List<Range> ranges = new ArrayList<>();
                ranges.add(zRangeUse);
                paramVar = paramVar.section(ranges);
              }
            }
            varNameList.add(paramVar.getFullName());
            varsSize += paramVar.getSize() * paramVar.getElementSize();
            varList.add(paramVar);
          }

        }
      }
    }

    return varsSize;
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.nexrad2;

import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.DiskCache;
import ucar.nc2.NetcdfFile;
import static ucar.nc2.iosp.nexrad2.Level2Record.REFLECTIVITY_HIGH;
import static ucar.nc2.iosp.nexrad2.Level2Record.VELOCITY_HIGH ;
import java.io.*;
import java.util.*;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;

import ucar.unidata.io.bzip2.CBZip2InputStream;
import ucar.unidata.io.bzip2.BZip2ReadException;

/**
 * This class reads a NEXRAD level II data file.
 * It can handle NCDC archives (ARCHIVE2), as well as CRAFT/IDD compressed files (AR2V0001).
 *

 * Adapted with permission from the Java Iras software developed by David Priegnitz at NSSL.

 *

 * Documentation on Archive Level II data format can be found at:
 *
 * http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html
 *
 * @author caron
 * @author David Priegnitz
 */
public class Level2VolumeScan {

  // data formats
  static public final String ARCHIVE2 = "ARCHIVE2";
  static public final String AR2V0001 = "AR2V0001";
  static public final String AR2V0002 = "AR2V0002";
  static public final String AR2V0003 = "AR2V0003";
  static public final String AR2V0004 = "AR2V0004";
  static public final String AR2V0006 = "AR2V0006";
  static public final String AR2V0007 = "AR2V0007";

  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Level2VolumeScan.class);
  ////////////////////////////////////////////////////////////////////////////////////

  // Data file
  RandomAccessFile raf;

  private String dataFormat = null; // ARCHIVE2 or AR2V0001
  private int title_julianDay; // days since 1/1/70
  private int title_msecs; // milliseconds since midnight
  private String stationId; // 4 letter station assigned by ICAO
  private NexradStationDB.Station station; // from lookup table, may be null
  private Level2Record first, last;

  private int vcp = 0; // Volume coverage pattern
  private int max_radials = 0;
  private int min_radials = Integer.MAX_VALUE;
  private int max_radials_hr = 0;
  private int min_radials_hr = Integer.MAX_VALUE;
  private int dopplarResolution;
  private boolean hasDifferentDopplarResolutions;
  private boolean hasHighResolutionData;

  private boolean hasHighResolutionREF;
  private boolean hasHighResolutionVEL;
  private boolean hasHighResolutionSW;
  private boolean hasHighResolutionZDR;
  private boolean hasHighResolutionPHI;
  private boolean hasHighResolutionRHO;

  private List<List<Level2Record>> reflectivityGroups, dopplerGroups;
  private List<List<Level2Record>> reflectivityHighResGroups;
  private List<List<Level2Record>> velocityHighResGroups;
  private List<List<Level2Record>> spectrumHighResGroups;

  private List<List<Level2Record>> diffReflectHighResGroups;
  private List<List<Level2Record>> diffPhaseHighResGroups;
  private List<List<Level2Record>> coefficientHighResGroups;

  private boolean showMessages = false, showData = false, debugScans = false, debugGroups2 = false, debugRadials = false;
  private boolean runCheck = false;

  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled())
      log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(4);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else
        station = NexradStationDB.get(stationId);
    }

    //see if we have to uncompress
     if (dataFormat.equals(AR2V0001) || dataFormat.equals(AR2V0003)
            || dataFormat.equals(AR2V0004) || dataFormat.equals(AR2V0006)  || dataFormat.equals(AR2V0007) ) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf = null;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          try (FileInputStream fstream = new FileInputStream(uncompressedFile)) {
            //lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } // Lock is released when the corresponding channel is closed on exit of try()
          uraf = ucar.unidata.io.RandomAccessFile.acquire(uncompressedFile.getPath());

        } else {
          try {
            // nope, gotta uncompress it
            uraf = uncompress(raf, uncompressedFile.getPath());
            if (log.isDebugEnabled()) log.debug("made uncompressed file= " + uncompressedFile.getPath());
          } catch (Throwable t) {
            if (uraf != null) uraf.close();
            throw t;
          }
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<>();
    List<Level2Record> doppler = new ArrayList<>();
    List<Level2Record> highReflectivity = new ArrayList<>();
    List<Level2Record> highVelocity = new ArrayList<>();
    List<Level2Record> highSpectrum = new ArrayList<>();
    List<Level2Record> highDiffReflectivity = new ArrayList<>();
    List<Level2Record> highDiffPhase = new ArrayList<>();
    List<Level2Record> highCorreCoefficient = new ArrayList<>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData)
        reflectivity.add(r);
      if (r.hasDopplerData)
        doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData)
          highReflectivity.add(r);
        if (r.hasHighResVELData)
          highVelocity.add(r);
        if (r.hasHighResSWData)
          highSpectrum.add(r);
        if (r.hasHighResZDRData)
          highDiffReflectivity.add(r);
        if (r.hasHighResPHIData)
          highDiffPhase.add(r);
        if (r.hasHighResRHOData)
          highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials) System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);

  }

  private List<List<Level2Record>> sortScans(String name, List<Level2Record> scans, int siz) {

    // now group by elevation_num
    Map<Short, List<Level2Record>> groupHash = new HashMap<>(siz);
    for (Level2Record record : scans) {
      List<Level2Record> group = groupHash.get(record.elevation_num);
      if (null == group) {
        group = new ArrayList<>();
        groupHash.put(record.elevation_num, group);
      }
      group.add(record);
    }

    // sort the groups by elevation_num
    List<List<Level2Record>> groups = new ArrayList<>(groupHash.values());
    Collections.sort(groups, new GroupComparator());

    // use the maximum radials
    for (List<Level2Record> group: groups) {
      Level2Record r = group.get(0);
      if(runCheck) testScan(name, group);
      if(r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) {
          if(group.size() <= 360) {
              max_radials = Math.max(max_radials, group.size());
              min_radials = Math.min(min_radials, group.size());
          } else {
              max_radials_hr = Math.max(max_radials_hr, group.size());
              min_radials_hr = Math.min(min_radials_hr, group.size());
          }
      }
      else {
        max_radials = Math.max(max_radials, group.size());
        min_radials = Math.min(min_radials, group.size());
      }
    }

    if (debugRadials) {
      System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
      for (List<Level2Record> group: groups) {
        Level2Record lastr = group.get(0);
        for (int j = 1; j < group.size(); j++) {
          Level2Record r = group.get(j);
          if (r.data_msecs < lastr.data_msecs)
            System.out.println(" out of order " + j);
          lastr = r;
        }
      }
    }

    testVariable(name, groups);
    if (debugScans) System.out.println("-----------------------------");

    return groups;
  }

  public int getMaxRadials(int r) {
    if (r == 0)
      return max_radials;
    else if (r == 1)
      return max_radials_hr;
    else
      return 0;
  }

  public int getMinRadials(int r) {
    if (r == 0)
      return min_radials;
    else if (r == 1)
      return min_radials_hr;
    else
      return 0;
  }

  public int getDopplarResolution() {
    return dopplarResolution;
  }

  public boolean hasDifferentDopplarResolutions() {
    return hasDifferentDopplarResolutions;
  }

  public boolean hasHighResolutions(int dt) {
    if (dt == 0)
      return hasHighResolutionData;
    else if (dt == 1)
      return hasHighResolutionREF;
    else if (dt == 2)
      return hasHighResolutionVEL;
    else if (dt == 3)
      return hasHighResolutionSW;
    else if (dt == 4)
      return hasHighResolutionZDR;
    else if (dt == 5)
      return hasHighResolutionPHI;
    else if (dt == 6)
      return hasHighResolutionRHO;
    else
      return false;
  }

  // do we have same characteristics for all records in a scan?
  private int MAX_RADIAL = 721;
  private int[] radial = new int[MAX_RADIAL];

  private boolean testScan(String name, List<Level2Record> group) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    Level2Record first = group.get(0);

    int n = group.size();
    if (debugScans) {
      boolean hasBoth = first.hasDopplerData && first.hasReflectData;
      System.out.println(name + " " + first + " has " + n + " radials resolution= " + first.resolution + " has both = " + hasBoth);
    }

    boolean ok = true;

    for (int i = 0; i < MAX_RADIAL; i++)
      radial[i] = 0;

    for (Level2Record r: group) {
      /* this appears to be common - seems to be ok, we put missing values in
      if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
        log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
                "!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
        ok = false;
      } */

      if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
        log.warn(raf.getLocation() + " different gate size (" + r.getGateSize(datatype) + ") in record " + name + " " + r);
        ok = false;
      }
      if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
        log.warn(raf.getLocation() + " different gate start (" + r.getGateStart(datatype) + ") in record " + name + " " + r);
        ok = false;
      }
      if (r.resolution != first.resolution) {
        log.warn(raf.getLocation() + " different resolution (" + r.resolution + ") in record " + name + " " + r);
        ok = false;
      }

      if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
        log.info(raf.getLocation() + " radial out of range= " + r.radial_num + " in record " + name + " " + r);
        continue;
      }
      if (radial[r.radial_num] > 0) {
        log.warn(raf.getLocation() + " duplicate radial = " + r.radial_num + " in record " + name + " " + r);
        ok = false;
      }
      radial[r.radial_num] = r.recno + 1;
    }

    for (int i = 1; i < radial.length; i++) {
      if (0 == radial[i]) {
        if (n != (i - 1)) {
          log.warn(" missing radial(s)");
          ok = false;
        }
        break;
      }
    }

    return ok;
  }

  // do we have same characteristics for all groups in a variable?
  private boolean testVariable(String name, List scans) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    if (scans.size() == 0) {
      log.warn(" No data for = " + name);
      return false;
    }

    boolean ok = true;
    List firstScan = (List) scans.get(0);
    Level2Record firstRecord = (Level2Record) firstScan.get(0);
    dopplarResolution = firstRecord.resolution;

    if (debugGroups2)
      System.out.println("Group " + Level2Record.getDatatypeName(datatype) + " ngates = " + firstRecord.getGateCount(datatype) +
              " start = " + firstRecord.getGateStart(datatype) + " size = " + firstRecord.getGateSize(datatype));

    for (int i = 1; i < scans.size(); i++) {
      List scan = (List) scans.get(i);
      Level2Record record = (Level2Record) scan.get(0);

      if ((datatype == Level2Record.VELOCITY_HI) && (record.resolution != firstRecord.resolution)) { // do all velocity resolutions match ??
        log.warn(name + " scan " + i + " diff resolutions = " + record.resolution + ", " + firstRecord.resolution +
                " elev= " + record.elevation_num + " " + record.getElevation());
        ok = false;
        hasDifferentDopplarResolutions = true;
      }

      if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
        log.warn(name + " scan " + i + " diff gates size = " + record.getGateSize(datatype) + " " + firstRecord.getGateSize(datatype) +
                " elev= " + record.elevation_num + " " + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(" ok gates size elev= " + record.elevation_num + " " + record.getElevation());

      if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
        log.warn(name + " scan " + i + " diff gates start = " + record.getGateStart(datatype) + " " + firstRecord.getGateStart(datatype) +
                " elev= " + record.elevation_num + " " + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(" ok gates start elev= " + record.elevation_num + " " + record.getElevation());

      if (record.message_type == 31) {
        hasHighResolutionData = true;
        //each data type
        if (record.hasHighResREFData)
          hasHighResolutionREF = true;
        if (record.hasHighResVELData)
          hasHighResolutionVEL = true;
        if (record.hasHighResSWData)
          hasHighResolutionSW = true;
        if (record.hasHighResZDRData)
          hasHighResolutionZDR = true;
        if (record.hasHighResPHIData)
          hasHighResolutionPHI = true;
        if (record.hasHighResRHOData)
          hasHighResolutionRHO = true;

      }
    }

    return ok;
  }

  /**
   * Get Reflectivity Groups
   * Groups are all the records for a variable and elevation_num;
   *
   * @return List of type List of type Level2Record
   */
  public List<List<Level2Record>> getReflectivityGroups() {
    return reflectivityGroups;
  }

  /**
   * Get Velocity Groups
   * Groups are all the records for a variable and elevation_num;
   *
   * @return List of type List of type Level2Record
   */
  public List<List<Level2Record>> getVelocityGroups() {
    return dopplerGroups;
  }

  public List<List<Level2Record>> getHighResVelocityGroups() {
    return velocityHighResGroups;
  }

  public List<List<Level2Record>> getHighResReflectivityGroups() {
    return reflectivityHighResGroups;
  }

  public List<List<Level2Record>> getHighResSpectrumGroups() {
    return spectrumHighResGroups;
  }

  public List<List<Level2Record>> getHighResDiffReflectGroups() {
    return diffReflectHighResGroups;
  }

  public List<List<Level2Record>> getHighResDiffPhaseGroups() {
    return diffPhaseHighResGroups;
  }

  public List<List<Level2Record>> getHighResCoeffocientGroups() {
    return coefficientHighResGroups;
  }

  private static class GroupComparator implements
    Comparator<List<Level2Record>> {

    public int compare(List<Level2Record> group1, List<Level2Record> group2) {
      Level2Record record1 = group1.get(0);
      Level2Record record2 = group2.get(0);

      //if (record1.elevation_num != record2.elevation_num)
      return record1.elevation_num - record2.elevation_num;
      //return record1.cut - record2.cut;
    }
  }

  /**
   * Get data format (ARCHIVE2, AR2V0001) for this file.
   *
   * @return data format (ARCHIVE2, AR2V0001) for this file.
   */
  public String getDataFormat() {
    return dataFormat;
  }

  /**
   * Get the starting Julian day for this volume
   *
   * @return days since 1/1/70.
   */
  public int getTitleJulianDays() {
    return title_julianDay;
  }

  /**
   * Get the starting time in seconds since midnight.
   *
   * @return Generation time of data in milliseconds of day past  midnight (UTC).
   */
  public int getTitleMsecs() {
    return title_msecs;
  }

  /**
   * Get the Volume Coverage Pattern number for this data.
   *
   * @return VCP
   * @see Level2Record#getVolumeCoveragePatternName
   */
  public int getVCP() {
    return vcp;
  }

  /**
   * Get the 4-char station ID for this data
   *
   * @return station ID (may be null)
   */

  public String getStationId() {
    return stationId;
  }

  public String getStationName() {
    return station == null ? "unknown" : station.name;
  }

  public double getStationLatitude() {
    return station == null ? 0.0 : station.lat;
  }

  public double getStationLongitude() {
    return station == null ? 0.0 : station.lon;
  }

  public double getStationElevation() {
    return station == null ? 0.0 : station.elev;
  }

  public Date getStartDate() {
    return first.getDate();
  }

  public Date getEndDate() {
    return last.getDate();
  }

  /**
   * Write equivilent uncompressed version of the file.
   *
   * @param inputRaf  file to uncompress
   * @param ufilename write to this file
   * @return raf of uncompressed file
   * @throws IOException on read error
   */
  private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename) throws IOException {
    RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
    FileLock lock;

    while (true) { // loop waiting for the lock
      try {
        lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
        break;

      } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
        try {
          Thread.sleep(100); // msecs
        } catch (InterruptedException e1) {
        }
      } catch (IOException e) {
          outputRaf.close();
          throw e;
      }
    }

    try {
        inputRaf.seek(0);
        byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
        int bytesRead = inputRaf.read(header);
        if (bytesRead != header.length)
        {
            throw new IOException("Error reading NEXRAD2 header -- got " +
                    bytesRead + " rather than" + header.length);
        }
        outputRaf.write(header);

        boolean eof = false;
        int numCompBytes;
        byte[] ubuff = new byte[40000];
        byte[] obuff = new byte[40000];

        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
            try {
                numCompBytes = inputRaf.readInt();
                if (numCompBytes == -1) {
                    if (log.isDebugEnabled())
                        log.debug("  done: numCompBytes=-1 ");
                    break;
                }
            } catch (EOFException ee) {
                log.debug("got EOFException");
                break; // assume this is ok
            }

            if (log.isDebugEnabled()) {
                log.debug("reading compressed bytes " + numCompBytes + " input starts at " + inputRaf.getFilePointer() + "; output starts at " + outputRaf.getFilePointer());

            }
          /*
          * For some stupid reason, the last block seems to
          * have the number of bytes negated.  So, we just
          * assume that any negative number (other than -1)
          * is the last block and go on our merry little way.
          */
            if (numCompBytes < 0) {
                if (log.isDebugEnabled())
                    log.debug("last block?" + numCompBytes);
                numCompBytes = -numCompBytes;
                eof = true;
            }
            byte[] buf = new byte[numCompBytes];
            inputRaf.readFully(buf);
            ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2,
                    numCompBytes - 2);

            //CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
            cbzip2.setStream(bis);
            int total = 0;
            int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
            try {
                while ((nread = cbzip2.read(ubuff)) != -1) {
                    if (total + nread > obuff.length) {
                        byte[] temp = obuff;
                        obuff = new byte[temp.length * 2];
                        System.arraycopy(temp, 0, obuff, 0, temp.length);
                    }
                    System.arraycopy(ubuff, 0, obuff, total, nread);
                    total += nread;
                }
                if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
            } catch (BZip2ReadException ioe) {
                log.warn("Nexrad2IOSP.uncompress ", ioe);
            }
            float nrecords = (float) (total / 2432.0);
            if (log.isDebugEnabled())
                log.debug("  unpacked " + total + " num bytes " + nrecords + " records; ouput ends at " + outputRaf.getFilePointer());
        }

        outputRaf.flush();
    } catch (IOException e) {
        if (outputRaf != null) outputRaf.close();

        // dont leave bad files around
        File ufile = new File(ufilename);
        if (ufile.exists()) {
            if (!ufile.delete())
                log.warn("failed to delete uncompressed file (IOException)" + ufilename);
        }

        throw e;
    } finally {
      try {
          if (lock != null) lock.release();
      } catch (IOException e) {
          if (outputRaf != null) outputRaf.close();
          throw e;
      }
    }

    return outputRaf;
  }

  // check if compressed file seems ok
  static public long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    try (RandomAccessFile raf = RandomAccessFile.acquire(ufilename)) {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);
      String test = raf.readString(8);
      if (test.equals(Level2VolumeScan.ARCHIVE2) ||
                test.startsWith("AR2V000")) {
        System.out.println("--Good header= " + test);
        raf.seek(24);
      } else {
        System.out.println("--No header ");
        lookForHeader = true;
        raf.seek(0);
      }

      boolean eof = false;
      int numCompBytes;

      while (!eof) {

        if (lookForHeader) {
          test = raf.readString(8);
          if (test.equals(Level2VolumeScan.ARCHIVE2) ||
                  test.startsWith("AR2V000")) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
      return raf.getFilePointer();
    } catch (EOFException e) {
      e.printStackTrace();
    }
    return 0;
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.grib.grib1;

import javax.annotation.Nullable;
import ucar.nc2.grib.GribNumbers;
import ucar.unidata.io.KMPMatch;
import ucar.unidata.io.RandomAccessFile;
import ucar.unidata.util.StringUtil2;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

/**
 * Scan files and extract Grib1Records. usage:
 * <pre>
 * Grib1RecordScanner reader = new Grib1RecordScanner(raf);
 * while (reader.hasNext()) {
 * ucar.nc2.grib.grib1.Grib1Record gr = reader.next();
 * Grib1SectionProductDefinition pds = gr.getPDSsection();
 * Grib1SectionGridDefinition gds = gr.getGDSsection();
 * ...
 * }
 *
 * </pre>
 *
 * @author John
 * @since 9/3/11
 */
public class Grib1RecordScanner {

  private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Grib1RecordScanner.class);
  private static final KMPMatch matcher = new KMPMatch(new byte[]{'G', 'R', 'I', 'B'});
  private static final boolean debug = false;
  private static final boolean debugGds = false;
  private static final int maxScan = 16000;

  static boolean allowBadIsLength = false;
  static boolean allowBadDsLength = false; // ECMWF workaround

  public static void setAllowBadIsLength(boolean allowBadIsLength) {
    Grib1RecordScanner.allowBadIsLength = allowBadIsLength;
  }

  public static void setAllowBadDsLength(boolean allowBadDsLength) {
    Grib1RecordScanner.allowBadDsLength = allowBadDsLength;
  }

  public static boolean isValidFile(RandomAccessFile raf) {
    try {
      raf.seek(0);
      boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
      if (!found) {
        return false;
      }
      raf.skipBytes(4); // will be positioned on byte 0 of indicator section
      int len = GribNumbers.uint3(raf);
      int edition = raf.read(); // read at byte 8
      if (edition != 1) {
        return false;
      }

      /* Due to a trick done by ECMWF's GRIBEX to support large GRIBs, we need a special treatment
       * to fix the length of the GRIB message. See:
       * https://software.ecmwf.int/wiki/display/EMOS/Changes+in+cycle+000281
       * https://github.com/Unidata/thredds/issues/445
       */
      len = getFixedTotalLengthEcmwfLargeGrib(raf, len);

      // check ending = 7777
      if (len > raf.length()) {
        return false;
      }
      if (allowBadIsLength) {
        return true;
      }

      raf.skipBytes(len - 12);
      for (int i = 0; i < 4; i++) {
        if (raf.read() != 55) {
          return false;
        }
      }
      return true;

    } catch (IOException e) {
      return false;
    }
  }

  static int getFixedTotalLengthEcmwfLargeGrib(RandomAccessFile raf, int len) throws IOException {
    int lenActual = len;
    if ((len & 0x800000) == 0x800000) {
      long pos0 = raf.getFilePointer(); // remember the actual pos
      int lenS1 = GribNumbers.uint3(raf); // section1Length
      raf.skipBytes(1); // table2Version
      if (GribNumbers.uint(raf) == 98) { // center (if ECMWF make the black magic)
        raf.skipBytes(2); // generatingProcessIdentifier, gridDefinition
        int s1f = GribNumbers.uint(raf); // section1Flags
        raf.skipBytes(lenS1 - (3 + 5)); // skips to next section
        int lenS2;
        int lenS3;
        if ((s1f & 128) == 128) { // section2 GDS exists
          lenS2 = GribNumbers.uint3(raf); // section2Length
          raf.skipBytes(lenS2 - 3); // skips to next section
        }
        if ((s1f & 64) == 64) { // section3 BMS exists
          lenS3 = GribNumbers.uint3(raf); // section3Length
          raf.skipBytes(lenS3 - 3); // skips to next section
        }
        int lenS4 = GribNumbers.uint3(raf); // section4Length
        if (lenS4 < 120) { // here we are!!!!
          lenActual = (len & 0x7FFFFF) * 120 - lenS4 + 4; // the actual totalLength
        }
      }
      raf.seek(pos0); // recall the pos
    }
    return lenActual;
  }

////////////////////////////////////////////////////////////

  private final Map<Long, Grib1SectionGridDefinition> gdsMap = new HashMap<>();
  private final ucar.unidata.io.RandomAccessFile raf;

  private byte[] header;
  private long lastPos;

  public Grib1RecordScanner(RandomAccessFile raf) throws IOException {
    this.raf = raf;
    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);
    lastPos = 0;
  }

  public boolean hasNext() throws IOException {
    if (lastPos >= raf.length()) {
      return false;
    }
    boolean more;
    long foundAt = 0;

    while (true) { // scan until we get a GRIB-1 or more is false
      raf.seek(lastPos);
      more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
      if (!more) {
        break;
      }

      foundAt = raf.getFilePointer();
      // see if its GRIB-1
      raf.skipBytes(7);
      int edition = raf.read();
      if (edition == 1) {
        break;
      }
      lastPos = raf.getFilePointer(); // not edition 1 ! could terminate ??
    }

    if (more) {
      // read the header - stuff between the records
      int sizeHeader = (int) (foundAt - lastPos);
      if (sizeHeader > 100) {
        sizeHeader = 100;   // maximum 100 bytes, more likely to be garbage
      }
      long startPos = foundAt - sizeHeader;
      header = new byte[sizeHeader];
      raf.seek(startPos);
      raf.readFully(header);
      raf.seek(foundAt);
      this.lastPos = foundAt; // ok start from here next time
    }

    return more;
  }

  @Nullable
  public Grib1Record next() throws IOException {

    Grib1SectionIndicator is = null;
    try {
      is = new Grib1SectionIndicator(raf);
      Grib1SectionProductDefinition pds = new Grib1SectionProductDefinition(raf);
      Grib1SectionGridDefinition gds = pds.gdsExists() ? new Grib1SectionGridDefinition(raf)
          : new Grib1SectionGridDefinition(pds);
      if (!pds.gdsExists() && debugGds) {
        log.warn(" NO GDS: center = %d, GridDefinition=%d file=%s%n", pds.getCenter(),
            pds.getGridDefinition(), raf.getLocation());
      }

      Grib1SectionBitMap bitmap = pds.bmsExists() ? new Grib1SectionBitMap(raf) : null;
      Grib1SectionBinaryData dataSection = new Grib1SectionBinaryData(raf);

      long ending = is.getEndPos();
      long dataEnding = dataSection.getStartingPosition() + dataSection.getLength();

      if (dataEnding > is.getEndPos()) { // presumably corrupt
        // raf.seek(dataSection.getStartingPosition()); // go back to start of the dataSection, in hopes of salvaging
        log.warn("BAD GRIB-1 data message at " + dataSection.getStartingPosition() + " header= "
            + StringUtil2.cleanup(header) + " for=" + raf.getLocation());
        throw new IllegalStateException("Illegal Grib1SectionBinaryData Message Length");
      }

      /* ecmwf offset by 1 bug - LOOK not sure if this is still needed
          // obtain BMS or BDS offset in the file for this product
          if (pds.getPdsVars().getCenter() == 98) {  // check for ecmwf offset by 1 bug
            int length = GribNumbers.uint3(raf);  // should be length of BMS
            if ((length + raf.getFilePointer()) < EOR) {
              dataOffset = raf.getFilePointer() - 3;  // ok
            } else {
              //System.out.println("ECMWF off by 1 bug" );
              dataOffset = raf.getFilePointer() - 2;
            }
          } else {
            dataOffset = raf.getFilePointer();
          }
      */

      // look for duplicate gds
      long crc = gds.calcCRC();
      Grib1SectionGridDefinition gdsCached = gdsMap.get(crc);
      if (gdsCached != null) {
        gds = gdsCached;
      } else {
        gdsMap.put(crc, gds);
      }

      // check that end section is correct
      boolean foundEnding = checkEnding(ending);
      log.debug(" read until %d grib ending at %d header ='%s' foundEnding=%s%n",
            raf.getFilePointer(), ending, StringUtil2.cleanup(header), foundEnding);

      if (!foundEnding && (allowBadIsLength || is.isMessageLengthFixed)) {
        foundEnding = checkEnding(dataSection.getStartingPosition() + dataSection.getLength());
      }

      if (!foundEnding && (allowBadDsLength || is.isMessageLengthFixed)) {
        foundEnding = true;
      }

      if (foundEnding) {
        lastPos = raf.getFilePointer();
        return new Grib1Record(header, is, gds, pds, bitmap, dataSection);
      }

      // skip this record
      // lastPos = is.getEndPos() + 20;  cant use is.getEndPos(), may be bad
      lastPos += 20;  // skip over the "GRIB" of this message
      if (hasNext()) { // search forward for another one
        return next();
      }

    } catch (Throwable t) {
      long pos = (is == null) ? -1 : is.getStartPos();
      log.warn("Bad Grib1 record in file {}, skipping pos={}", raf.getLocation(), pos);
      // t.printStackTrace();
      lastPos += 20; // skip over the "GRIB"
      if (hasNext()) { // search forward for another one
        return next();
      }
    }

    // EOF that hasNext() couldnt detect
    return null;
  }

  private boolean checkEnding(long ending) throws IOException {
    // check that end section = "7777" is correct
    raf.seek(ending - 4);
    for (int i = 0; i < 4; i++) {
      if (raf.read() != 55) {
        String clean = StringUtil2.cleanup(header);
        if (clean.length() > 40) {
          clean = clean.substring(0, 40) + "...";
        }
        log.debug(
            "Missing End of GRIB message at pos=" + ending + " header= " + clean + " for=" + raf
                .getLocation());
        return false;
      }
    }
    return true;
  }

  // Count the number of records in a grib1 file.
  public static void main(String[] args) throws IOException {
    int count = 0;
    String file = (args.length > 0) ? args[0] : "Q:/cdmUnitTest/formats/grib1/ECMWF.hybrid.grib1";
    RandomAccessFile raf = new RandomAccessFile(file, "r");
    System.out.printf("Read %s%n", raf.getLocation());
    Grib1RecordScanner scan = new Grib1RecordScanner(raf);
    while (scan.hasNext()) {
      scan.next();
      count++;
    }
    raf.close();
    System.out.printf("count=%d%n", count);
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.units;

/**
 * Provides support for unit names.
 *
 * @author Steven R. Emmerson
 */
public final class UnitName extends UnitID implements Comparable<Object> {
	private static final long	serialVersionUID	= 1L;

	/**
	 * The name of the unit.
	 *
	 * @serial
	 */
	private final String		name;

	/**
	 * The plural form of the unit name.
	 *
	 * @serial
	 */
	private final String		plural;

	/**
	 * The symbol for the unit.
	 *
	 * @serial
	 */
	private final String		symbol;

	/**
	 * Constructs from a name and a symbol. Regular rules are use to construct
	 * the plural form of the unit name.
	 *
	 * @param name
	 *            The name of the unit. Shall not be
     *				null.
	 * @param symbol
	 *            The symbol for the unit. May be null
     *				.
	 * @throws NameException
	 *             name == null.
	 */
	protected UnitName(final String name, final String symbol)
			throws NameException {
		this(name, null, symbol);
	}

	/**
	 * Constructs from a name, a plural form of the unit name, and a symbol.
	 *
	 * @param name
	 *            The name of the unit. Shall not be
     *				null.
	 * @param plural
	 *            The plural form of the name. May be
     *				null, in which
	 *            case regular plural- forming rules are used to construct the
	 *            plural form from the name.
	 * @param symbol
	 *            The symbol for the unit. May be null
     *				.
	 * @throws NameException
	 *             name == null.
	 */
	protected UnitName(final String name, final String plural,
			final String symbol) throws NameException {
		if (name == null) {
			throw new NameException("Unit name can't be null");
		}
		this.name = name;
		this.plural = plural == null
				? makePlural(name)
				: plural;
		this.symbol = symbol;
	}

	/**
	 * Factory method for constructing a UnitName from a name.
	 *
	 * @param name
	 *            The name of the unit. Shall not be
     *				null.
	 * @throws NameException
	 *             name == null.
	 */
	public static UnitName newUnitName(final String name) throws NameException {
		return newUnitName(name, null);
	}

	/**
	 * Factory method for constructing a UnitName from a name and a plural form
	 * of the name.
	 *
	 * @param name
	 *            The name of the unit. Shall not be
     *				null.
	 * @param plural
	 *            The plural form of the name. May be
     *				null, in which
	 *            case regular plural- forming rules are used to construct the
	 *            plural form from the name.
	 * @throws NameException
	 *             name == null.
	 */
	public static UnitName newUnitName(final String name, final String plural)
			throws NameException {
		return newUnitName(name, plural, null);
	}

	/**
	 * Factory method for constructing a UnitName from a name, a plural form of
	 * the name, and a symbol.
	 *
	 * @param name
	 *            The name of the unit. Shall not be
     *				null.
	 * @param plural
	 *            The plural form of the name. May be
     *				null, in which
	 *            case regular plural- forming rules are used to construct the
	 *            plural form from the name.
	 * @param symbol
	 *            The symbol for the unit. May be null
     *				.
	 * @throws NameException
	 *             name == null.
	 */
	public static UnitName newUnitName(final String name, final String plural,
			final String symbol) throws NameException {
		return new UnitName(name, plural, symbol);
	}

	/**
	 * Returns the name.
	 *
	 * @return The name. Won't be null.
	 */
	@Override
	public final String getName() {
		return name;
	}

	/**
	 * Returns the plural form of the unit name.
	 *
	 * @return The plural form of the unit name.
	 */
	@Override
	public String getPlural() {
		return plural;
	}

	/**
	 * Returns the symbol.
	 *
	 * @return The symbol. Might be null.
	 */
	@Override
	public final String getSymbol() {
		return symbol;
	}

	/**
	 * Returns the string representation of this identifier.
	 *
	 * @return The string representation of this identifier.
	 */
	@Override
	public final String toString() {
		final String string = getSymbol();
		return string == null
				? getName()
				: string;
	}

	/**
	 * Compares this UnitName with another UnitName.
	 */
	public final int compareTo(final Object object) {
		return getName().compareToIgnoreCase(((UnitName) object).getName());
	}

	/**
	 * Indicates if this UnitName is semantically identical to an object.
	 */
	@Override
	public final boolean equals(final Object object) {
		return object instanceof UnitName && compareTo(object) == 0;
	}

	/**
	 * Returns the hash code of this instance.
	 *
	 * @return The hash code of this instance.
	 */
	@Override
	public int hashCode() {
		return getName().toLowerCase().hashCode();
	}

	/**
	 * Returns the plural form of a name. Regular rules are used to generate the
	 * plural form.
	 *
	 * @param name
	 *            The name.
	 * @return The plural form of the name.
	 */
	protected String makePlural(final String name) {
		String plural;
		final int length = name.length();
		final char lastChar = name.charAt(length - 1);
		if (lastChar != 'y') {
			plural = name
					+ (lastChar == 's' || lastChar == 'x' || lastChar == 'z'
							|| name.endsWith("ch")
							? "es"
							: "s");
		}
		else {
			if (length == 1) {
				plural = name + "s";
			}
			else {
				final char penultimateChar = name.charAt(length - 2);
				plural = (penultimateChar == 'a' || penultimateChar == 'e'
						|| penultimateChar == 'i' || penultimateChar == 'o' || penultimateChar == 'u')
						? name + "s"
						: name.substring(0, length - 1) + "ies";
			}
		}
		return plural;
	}
}