/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.dt.grid;

import ucar.nc2.dataset.CoordinateAxis2D;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dt.GridCoordSystem;
import ucar.ma2.ArrayDouble;
import ucar.ma2.MAMath;

import java.io.IOException;

/**
 * 2D Coordinate System has lat(x,y) and lon(x,y).
 * This class implements finding the index (i,j) from (lat, lon) coord.
 * This is for "one-off" computation, not a systematic lookup table for all points in a pixel array.
 * Hueristic search of the 2D space for the cell that contains the point.
 *
 * @author caron
 * @since Jul 10, 2009
 */

public class GridCoordinate2D {
  static private boolean debug = false;
  static private final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GridCoordinate2D.class);

  static final String testfilename = "Q:/cdmUnitTest/ft/fmrc/rtofs/ofs.20091122/ofs_atl.t00z.F024.grb.grib2";

  private final CoordinateAxis2D latCoord, lonCoord;
  private final int nrows, ncols;
  private ArrayDouble.D2 latEdge, lonEdge;
  private MAMath.MinMax latMinMax, lonMinMax;

  GridCoordinate2D(CoordinateAxis2D latCoord, CoordinateAxis2D lonCoord) {
    this.latCoord = latCoord;
    this.lonCoord = lonCoord;

    assert latCoord.getRank() == 2;
    assert lonCoord.getRank() == 2;
    int[] shape = latCoord.getShape();

    nrows = shape[0];
    ncols = shape[1];
  }

  private void findBounds() {
    if (lonMinMax != null) return;

    lonEdge = lonCoord.getEdges();
    latEdge = latCoord.getEdges();

    // assume missing values have been converted to NaNs
    latMinMax = MAMath.getMinMax(latEdge);
    lonMinMax = MAMath.getMinMax(lonEdge);

    if (debug)
      System.out.printf("Bounds (%d %d): lat= (%f,%f) lon = (%f,%f) %n", nrows, ncols, latMinMax.min, latMinMax.max, lonMinMax.min, lonMinMax.max);
  }

  // brute force
  public boolean findCoordElementForce(double wantLat, double wantLon, int[] rectIndex) {
    findBounds();
    if (wantLat < latMinMax.min) return false;
    if (wantLat > latMinMax.max) return false;
    if (wantLon < lonMinMax.min) return false;
    if (wantLon > lonMinMax.max) return false;

    boolean saveDebug = debug;
    debug = false;
    for (int row=0; row<nrows; row++) {
      for (int col=0; col<ncols; col++) {

        rectIndex[0] = row;
        rectIndex[1] = col;

        if (contains(wantLat, wantLon, rectIndex)) {
          debug = saveDebug;
          return true;
        }
      }
    }
    //debug = saveDebug;
    return false;
  }

  public boolean findCoordElement(double wantLat, double wantLon, int[] rectIndex) {
    return findCoordElementNoForce(wantLat, wantLon,rectIndex);
  }

  /**
   * Find the best index for the given lat,lon point.
   * @param wantLat   lat of point
   * @param wantLon   lon of point
   * @param rectIndex return (row,col) index, or best guess here. may not be null
   *
   * @return false if not in the grid.
   */
  public boolean findCoordElementNoForce(double wantLat, double wantLon, int[] rectIndex) {
    findBounds();
    if (wantLat < latMinMax.min) return false;
    if (wantLat > latMinMax.max) return false;
    if (wantLon < lonMinMax.min) return false;
    if (wantLon > lonMinMax.max) return false;

    double gradientLat = (latMinMax.max - latMinMax.min) / nrows;
    double gradientLon = (lonMinMax.max - lonMinMax.min) / ncols;

    double diffLat = wantLat - latMinMax.min;
    double diffLon = wantLon - lonMinMax.min;

    // initial guess
    rectIndex[0] = (int) Math.round(diffLat / gradientLat);  // row
    rectIndex[1] =(int) Math.round(diffLon / gradientLon);  // col

    int count = 0;
    while (true) {
      count++;
      if (debug) System.out.printf("%nIteration %d %n", count);
      if (contains(wantLat, wantLon, rectIndex))
        return true;

      if (!jump2(wantLat, wantLon, rectIndex)) return false;

      // bouncing around
      if (count > 10) {
        // last ditch attempt
        return incr(wantLat, wantLon, rectIndex);
        //if (!ok)
        //  log.error("findCoordElement didnt converge lat,lon = "+wantLat+" "+ wantLon);
        //return ok;
      }
    }
  }

  /**
   * Is the point (lat,lon) contained in the (row, col) rectangle ?
   *
   * @param wantLat   lat of point
   * @param wantLon   lon of point
   * @param rectIndex rectangle row, col, will be clipped to [0, nrows), [0, ncols)
   * @return true if contained
   */
  private boolean containsOld(double wantLat, double wantLon, int[] rectIndex) {
    rectIndex[0] = Math.max( Math.min(rectIndex[0], nrows-1), 0);
    rectIndex[1] = Math.max( Math.min(rectIndex[1], ncols-1), 0);

    int row = rectIndex[0];
    int col = rectIndex[1];

    if (debug) System.out.printf(" (%d,%d) contains (%f,%f) in (lat=%f %f) (lon=%f %f) ?%n",
            rectIndex[0], rectIndex[1], wantLat, wantLon,
            latEdge.get(row, col), latEdge.get(row + 1, col), lonEdge.get(row, col), lonEdge.get(row, col + 1));

    if (wantLat < latEdge.get(row, col)) return false;
    if (wantLat > latEdge.get(row + 1, col)) return false;
    if (wantLon < lonEdge.get(row, col)) return false;
    if (wantLon > lonEdge.get(row, col + 1)) return false;
    return true;
  }

  /**
   * Is the point (lat,lon) contained in the (row, col) rectangle ?
   *
   * @param wantLat   lat of point
   * @param wantLon   lon of point
   * @param rectIndex rectangle row, col, will be clipped to [0, nrows), [0, ncols)
   * @return true if contained
   */

/*
  http://mathforum.org/library/drmath/view/54386.html

      Given any three points on the plane (x0,y0), (x1,y1), and
  (x2,y2), the area of the triangle determined by them is
  given by the following formula:

        1 | x0 y0 1 |
    A = - | x1 y1 1 |,
        2 | x2 y2 1 |

  where the vertical bars represent the determinant.
  the value of the expression above is:

       (.5)(x1*y2 - y1*x2 -x0*y2 + y0*x2 + x0*y1 - y0*x1)

  The amazing thing is that A is positive if the three points are
  taken in a counter-clockwise orientation, and negative otherwise.

  To be inside a rectangle (or any convex body), as you trace
  around in a clockwise direction from p1 to p2 to p3 to p4 and
  back to p1, the "areas" of triangles p1 p2 p, p2 p3 p, p3 p4 p,
  and p4 p1 p must all be positive.  If you don't know the
  orientation of your rectangle, then they must either all be
  positive or all be negative.

  this method works for arbitrary convex regions oo the plane.
*/
  private boolean contains(double wantLat, double wantLon, int[] rectIndex) {
    rectIndex[0] = Math.max( Math.min(rectIndex[0], nrows-1), 0);
    rectIndex[1] = Math.max( Math.min(rectIndex[1], ncols-1), 0);

    int row = rectIndex[0];
    int col = rectIndex[1];

    double x1 = lonEdge.get(row, col);
    double y1 = latEdge.get(row, col);

    double x2 = lonEdge.get(row, col+1);
    double y2 = latEdge.get(row, col+1);

    double x3 = lonEdge.get(row+1, col+1);
    double y3 = latEdge.get(row+1, col+1);

    double x4 = lonEdge.get(row+1, col);
    double y4 = latEdge.get(row+1, col);

    // must all have same determinate sign
    boolean sign = detIsPositive(x1, y1, x2, y2, wantLon, wantLat);
    if (sign != detIsPositive(x2, y2, x3, y3, wantLon, wantLat)) return false;
    if (sign != detIsPositive(x3, y3, x4, y4, wantLon, wantLat)) return false;
    if (sign != detIsPositive(x4, y4, x1, y1, wantLon, wantLat)) return false;

    return true;
  }

  private boolean detIsPositive(double x0, double y0, double x1, double y1, double x2, double y2) {
    double det = (x1*y2 - y1*x2 -x0*y2 + y0*x2 + x0*y1 - y0*x1);
    if (det == 0)
      log.warn("determinate = 0 on lat/lon=" + latCoord.getFullName() + ", " + latCoord.getFullName()); // LOOK needed?
    return det > 0;
  }

  private boolean jumpOld(double wantLat, double wantLon, int[] rectIndex) {
    int row = Math.max( Math.min(rectIndex[0], nrows-1), 0);
    int col = Math.max( Math.min(rectIndex[1], ncols-1), 0);
    double gradientLat = latEdge.get(row + 1, col) - latEdge.get(row, col);
    double gradientLon = lonEdge.get(row, col + 1) - lonEdge.get(row, col);

    double diffLat = wantLat - latEdge.get(row, col);
    double diffLon = wantLon - lonEdge.get(row, col);

    int drow = (int) Math.round(diffLat / gradientLat);
    int dcol = (int) Math.round(diffLon / gradientLon);

    if (debug) System.out.printf("   jump from %d %d (grad=%f %f) (diff=%f %f) (delta=%d %d)",
            row, col, gradientLat, gradientLon,
            diffLat, diffLon, drow, dcol);

    if ((drow == 0) && (dcol == 0)) {
      if (debug) System.out.printf("%n   incr:");
      return incr(wantLat, wantLon, rectIndex);
    } else {
      rectIndex[0] = Math.max( Math.min(row + drow, nrows-1), 0);
      rectIndex[1] = Math.max( Math.min(col + dcol, ncols-1), 0);
      if (debug) System.out.printf(" to (%d %d)%n", rectIndex[0], rectIndex[1]);
      if ((row == rectIndex[0]) && (col == rectIndex[1])) return false; // nothing has changed
    }

    return true;
  }

  /**
   * jump to a new row,col
   * @param wantLat want this lat
   * @param wantLon want this lon
   * @param rectIndex starting row, col and replaced by new guess
   * @return true if new guess, false if failure
   */
  /*
    choose x, y such that (matrix multiply) :

    (wantx) = (fxx fxy)  (x)
    (wanty)   (fyx fyy)  (y)

     where fxx = d(fx)/dx  ~= delta lon in lon direction
     where fxy = d(fx)/dy  ~= delta lon in lat direction
     where fyx = d(fy)/dx  ~= delta lat in lon direction
     where fyy = d(fy)/dy  ~= delta lat in lat direction

    2 linear equations in 2 unknowns, solve in usual way
   */
  private boolean jump2(double wantLat, double wantLon, int[] rectIndex) {
    int row = Math.max( Math.min(rectIndex[0], nrows-1), 0);
    int col = Math.max( Math.min(rectIndex[1], ncols-1), 0);
    double lat = latEdge.get(row, col);
    double lon =  lonEdge.get(row, col);
    double diffLat = wantLat - lat;
    double diffLon = wantLon - lon;

    double dlatdy = latEdge.get(row + 1, col) - lat;
    double dlatdx = latEdge.get(row, col+1) - lat;
    double dlondx = lonEdge.get(row, col + 1) - lon;
    double dlondy = lonEdge.get(row + 1, col) - lon;

    // solve for dlon

    double dx =  (diffLon - dlondy * diffLat / dlatdy) / (dlondx - dlatdx * dlondy / dlatdy);
    // double dy =  (diffLat - dlatdx * diffLon / dlondx) / (dlatdy - dlatdx * dlondy / dlondx);
    double dy =  (diffLat - dlatdx * dx) / dlatdy;

    if (debug) System.out.printf("   jump from %d %d (dlondx=%f dlondy=%f dlatdx=%f dlatdy=%f) (diffLat,Lon=%f %f) (deltalat,Lon=%f %f)",
            row, col, dlondx, dlondy, dlatdx, dlatdy,
            diffLat, diffLon, dy, dx);

    int drow = (int) Math.round(dy);
    int dcol = (int) Math.round(dx);

    if ((drow == 0) && (dcol == 0)) {
      if (debug) System.out.printf("%n   incr:");
      return incr(wantLat, wantLon, rectIndex);
    } else {
      rectIndex[0] = Math.max( Math.min(row + drow, nrows-1), 0);
      rectIndex[1] = Math.max( Math.min(col + dcol, ncols-1), 0);
      if (debug) System.out.printf(" to (%d %d)%n", rectIndex[0], rectIndex[1]);
      if ((row == rectIndex[0]) && (col == rectIndex[1])) return false; // nothing has changed
    }

    return true;
  }

  private boolean incr(double wantLat, double wantLon, int[] rectIndex) {
    int row = rectIndex[0];
    int col = rectIndex[1];
    double diffLat = wantLat - latEdge.get(row, col);
    double diffLon = wantLon - lonEdge.get(row, col);

    if (Math.abs(diffLat) > Math.abs(diffLon)) { // try lat first
      rectIndex[0] = row + ((diffLat > 0) ? 1 : -1);
      if (contains(wantLat, wantLon,  rectIndex)) return true;
      rectIndex[1] = col + ((diffLon > 0) ? 1 : -1);
      if (contains(wantLat, wantLon,  rectIndex)) return true;
    } else {
      rectIndex[1] = col + ((diffLon > 0) ? 1 : -1);
      if (contains(wantLat, wantLon,  rectIndex)) return true;
      rectIndex[0] = row + ((diffLat > 0) ? 1 : -1);
      if (contains(wantLat, wantLon,  rectIndex)) return true;
    }

    // back to original, do box search
    rectIndex[0] = row;
    rectIndex[1] = col;
    return box9(wantLat, wantLon, rectIndex);
  }

  // we think its got to be in one of the 9 boxes around rectIndex
  private boolean box9(double wantLat, double wantLon, int[] rectIndex) {
    int row = rectIndex[0];
    int minrow = Math.max(row-1, 0);
    int maxrow = Math.min(row+1, nrows);

    int col = rectIndex[1];
    int mincol= Math.max(col-1, 0);
    int maxcol = Math.min(col+1, ncols);

    if (debug) System.out.printf("%n   box9:");
    for (int i=minrow; i<=maxrow; i++)
      for (int j=mincol; j<=maxcol; j++) {
        rectIndex[0] = i;
        rectIndex[1] = j;
        if (contains(wantLat, wantLon,  rectIndex)) return true;
      }

    return false;
  }

  private static void doOne(GridCoordinate2D g2d, double wantLat, double wantLon) {
    int[] result = new int[2];
    if (g2d.findCoordElementForce(wantLat, wantLon, result))
      System.out.printf("Brute (%f %f) == (%d %d) %n", wantLat, wantLon, result[0], result[1]);
    else {
      System.out.printf("Brute (%f %f) FAIL", wantLat, wantLon);
      return;
    }

    if (g2d.findCoordElement(wantLat, wantLon, result))
      System.out.printf("(%f %f) == (%d %d) %n", wantLat, wantLon, result[0], result[1]);
    else
      System.out.printf("(%f %f) FAIL %n", wantLat, wantLon);
    System.out.printf("----------------------------------------%n");

  }

  public static void test1() throws IOException {
    String filename = "D:/work/asaScience/EGM200_3.ncml";
    GridDataset gds = GridDataset.open(filename);
    GeoGrid grid = gds.findGridByName("u_wind");
    GridCoordSystem gcs = grid.getCoordinateSystem();
    CoordinateAxis lonAxis = gcs.getXHorizAxis();
    assert lonAxis instanceof CoordinateAxis2D;
    CoordinateAxis latAxis = gcs.getYHorizAxis();
    assert latAxis instanceof CoordinateAxis2D;

    GridCoordinate2D g2d = new GridCoordinate2D((CoordinateAxis2D) latAxis, (CoordinateAxis2D) lonAxis);
    doOne(g2d, 35.0, -6.0);
    doOne(g2d, 34.667302, -5.008376); // FAIL
    doOne(g2d, 34.667303, -6.394240);
    doOne(g2d, 36.6346, -5.0084);
    doOne(g2d, 36.6346, -6.394240);

    gds.close();
  }

  public static void test2() throws IOException {
    String filename = "C:/data/fmrc/apex_fmrc/Run_20091025_0000.nc";
    GridDataset gds = GridDataset.open(filename);
    GeoGrid grid = gds.findGridByName("temp");
    GridCoordSystem gcs = grid.getCoordinateSystem();
    CoordinateAxis lonAxis = gcs.getXHorizAxis();
    assert lonAxis instanceof CoordinateAxis2D;
    CoordinateAxis latAxis = gcs.getYHorizAxis();
    assert latAxis instanceof CoordinateAxis2D;

    GridCoordinate2D g2d = new GridCoordinate2D((CoordinateAxis2D) latAxis, (CoordinateAxis2D) lonAxis);
    doOne(g2d, 40.166959,-73.954234);

    gds.close();
  }

  public static void test3() throws IOException {
    String filename = testfilename;
    GridDataset gds = GridDataset.open(filename);
    GeoGrid grid = gds.findGridByName("Sea_Surface_Height_Relative_to_Geoid_surface");
    GridCoordSystem gcs = grid.getCoordinateSystem();
    CoordinateAxis lonAxis = gcs.getXHorizAxis();
    assert lonAxis instanceof CoordinateAxis2D;
    CoordinateAxis latAxis = gcs.getYHorizAxis();
    assert latAxis instanceof CoordinateAxis2D;

    GridCoordinate2D g2d = new GridCoordinate2D((CoordinateAxis2D) latAxis, (CoordinateAxis2D) lonAxis);
    doOne(g2d, -15.554099426977835, -0.7742870290336263);

    gds.close();
  }

  public static void main(String[] args) throws IOException {
    debug = true;
    test3();
  }

}


/* Copyright (c) 2005 Robert Alten Simons (info@cohort.com).
 * See the MIT/X-like license in LICENSE.txt.
 * For more information visit www.cohort.com or contact info@cohort.com.
 */
package ucar.nc2.ogc.erddap.util;

import ucar.nc2.constants.CDM;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

/**
 * StringArray is a thin shell over a String[] with methods like ArrayList's
 * methods; it extends PrimitiveArray.
 * All of the methods which add strings to StringArray (e.g., add()),
 * use String2.canonical(), to ensure that canonical Strings are stored (to save memory
 * if there are duplicates).
 *
 *
This class uses "" to represent a missing value (NaN).
 */
public class ErddapStringArray {
    /** The number of active values (which may be different from the array's capacity). */
    protected int size = 0;

    /**
     * This is the main data structure.
     * This should be private, but is public so you can manipulate it if you
     * promise to be careful.
     * Note that if the PrimitiveArray's capacity is increased,
     * the PrimitiveArray will use a different array for storage.
     */
    public String[] array;

    /**
     * A constructor for a capacity of 8 elements. The initial 'size' will be 0.
     */
    public ErddapStringArray() {
        array = new String[8];
    }

    /**
     * Reads the contents of {@code inputStream} into a StringArray. Each line of input will result in an element in
     * the array.
     *

     * The specified stream remains open after this method returns.
     *
     * @param inputStream  a stream with line-based content.
     * @param charset      the name of a supported {@link java.nio.charset.Charset charset}.
     * @return  a StringArray created from the stream.
     * @throws IOException  if an I/O error occurs
     */
    public static ErddapStringArray fromInputStream(InputStream inputStream, String charset) throws IOException {
        if (charset == null || charset.isEmpty()) {
            charset = CDM.UTF8;
        }
        InputStreamReader isr = new InputStreamReader(inputStream, charset);
        BufferedReader bufferedReader = new BufferedReader(isr);

        ErddapStringArray sa = new ErddapStringArray();
        for (String s; (s = bufferedReader.readLine()) != null; ) {
            sa.add(s);
        }

        // Do not call BufferedReader.close() here; that would close the underlying InputStream, which is the
        // responsibility of the client. This ought to be safe, as neither InputStreamReader nor BufferedReader hold any
        // resources that a call to close() would make free and which the garbage collector would not make free anyway.
        return sa;
    }

    /**
     * Return the number of elements in the array.
     *
     * @return the number of elements in the array.
     */
    public int size() {
        return size;
    }

    /**
     * This adds an item to the array (increasing 'size' by 1).
     *
     * @param value the value to be added to the array
     */
    public void add(String value) {
        if (size == array.length) //if we're at capacity
            ensureCapacity(size + 1L);
        array[size++] = value;
    }

    /**
     * This ensures that the capacity is at least 'minCapacity'.
     *
     * @param minCapacity the minimum acceptable capacity.
     *    minCapacity is type long, but >= Integer.MAX_VALUE will throw exception.
     */
    public void ensureCapacity(long minCapacity) {
        if (array.length < minCapacity) {
            //ensure minCapacity is < Integer.MAX_VALUE
            ErddapMath2.ensureArraySizeOkay(minCapacity, "StringArray");

            //caller may know exact number needed, so don't double above 2x current size
            int newCapacity = (int)Math.min(Integer.MAX_VALUE - 1, array.length + (long)array.length);
            if (newCapacity < minCapacity)
                newCapacity = (int)minCapacity; //safe since checked above

            String[] newArray = new String[newCapacity];
            System.arraycopy(array, 0, newArray, 0, size);
            array = newArray; //do last to minimize concurrency problems
        }
    }

    /**
     * This gets a specified element.
     *
     * @param index 0 ... size-1
     */
    public String get(int index) {
        if (index >= size)
            throw new IllegalArgumentException(ErddapString2.ERROR + " in StringArray.get: index (" +
                index + ") >= size (" + size + ").");
        return array[index];
    }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.dataset.conv;

import ucar.ma2.DataType;
import ucar.nc2.*;
import ucar.nc2.constants.*;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dataset.CoordinateAxis1D;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.util.CancelTask;

import java.io.IOException;

/**
 * HDF5-EOS AURA OMI
 *
 * @author John
 * @since 12/26/12
 * @see "http://aura.gsfc.nasa.gov/instruments/omi.html"
 */
public class HdfEosOmiConvention extends ucar.nc2.dataset.CoordSysBuilder {

/**
 *
 */
    public static boolean isMine(NetcdfFile ncfile) {
        if (!ncfile.getFileTypeId().equals("HDF5-EOS")) { return false; }

        final String typeName = ncfile.findAttValueIgnoreCase(null, CF.FEATURE_TYPE, null);
        if (typeName == null) { return false; }
        if (!typeName.equals(FeatureType.GRID.toString()) &&
                !typeName.equals(FeatureType.SWATH.toString())) { return false; }

    /*
 group: HDFEOS {

    group: ADDITIONAL {

      group: FILE_ATTRIBUTES {
        :InstrumentName = "OMI";
        :ProcessLevel = "3e";
        :GranuleMonth = 12; // int
        :GranuleDay = 14; // int
        :GranuleYear = 2005; // int
        :GranuleDayOfYear = 348; // int
        :TAI93At0zOfGranule = 4.08672005E8; // double
        :PGEVersion = "0.9.26";
        :StartUTC = "2005-12-13T18:00:00.000000Z";
        :EndUTC = "2005-12-15T06:00:00.000000Z";
        :Period = "Daily";
      }
    }
     */
        final Attribute instName = ncfile.findAttribute("/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/@InstrumentName");
        if (instName == null || !instName.getStringValue().equals("OMI"))  { return false; }

        final Attribute level = ncfile.findAttribute("/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/@ProcessLevel");
        if (level == null)  { return false; }

        return !(!level.getStringValue().startsWith("2") && !level.getStringValue().startsWith("3"));
  }

/**
 *
 */
    public HdfEosOmiConvention() {
        this.conventionName = "HDF5-EOS-OMI";
    }

  /*

  // level3
   group: HDFEOS {

    group: GRIDS {

      group: OMI_Column_Amount_O3 {
        dimensions:
          XDim = 1440;
          YDim = 720;

        group: Data_Fields {
          variables:
            float ColumnAmountO3(YDim=720, XDim=1440);
              :_FillValue = -1.2676506E30f; // float
              :Units = "DU";
              :Title = "Best Total Ozone Solution";
              :UniqueFieldDefinition = "TOMS-OMI-Shared";
              :ScaleFactor = 1.0; // double
              :Offset = 0.0; // double
              :ValidRange = 50.0f, 700.0f; // float
              :MissingValue = -1.2676506E30f; // float
              :_ChunkSize = 180, 180; // int

            float Reflectivity331(YDim=720, XDim=1440);
              :_FillValue = -1.2676506E30f; // float
              :Units = "%";
              :Title = "Effective Surface Reflectivity at 331 nm";
              :UniqueFieldDefinition = "TOMS-OMI-Shared";
              :ScaleFactor = 1.0; // double
              :Offset = 0.0; // double
              :ValidRange = -15.0f, 115.0f; // float
              :MissingValue = -1.2676506E30f; // float
              :_ChunkSize = 180, 180; // int

            float UVAerosolIndex(YDim=720, XDim=1440);
              :_FillValue = -1.2676506E30f; // float
              :Units = "NoUnits";
              :Title = "UV Aerosol Index";
              :UniqueFieldDefinition = "TOMS-OMI-Shared";
              :ScaleFactor = 1.0; // double
              :Offset = 0.0; // double
              :ValidRange = -30.0f, 30.0f; // float
              :MissingValue = -1.2676506E30f; // float
              :_ChunkSize = 180, 180; // int

        }
        // group attributes:
        :GCTPProjectionCode = 0; // int
        :Projection = "Geographic";
        :GridOrigin = "Center";
        :NumberOfLongitudesInGrid = 1440; // int
        :NumberOfLatitudesInGrid = 720; // int
      }
    }
  }
   */
    public void augmentDataset(NetcdfDataset ds, CancelTask cancelTask) throws IOException {
        final Attribute levelAtt = ds.findAttribute("/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/@ProcessLevel");
        if (levelAtt == null)  { return; }
        final int level = levelAtt.getStringValue().startsWith("2") ? 2 : 3;

        //Attribute time = ds.findAttribute("/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/@TAI93At0zOfGranule");

        if (level == 3) { augmentDataset3(ds); }
    }

/**
 *
 */
    private void augmentDataset3(NetcdfDataset ds) throws IOException {
        final Group grids = ds.findGroup("/HDFEOS/GRIDS");
        if (grids == null) { return; }
        for (final Group g2 : grids.getGroups()) {
            final Attribute gctp = g2.findAttribute("GCTPProjectionCode");

            if (gctp == null || !gctp.getNumericValue().equals(0)) { continue; }

            final Attribute nlon = g2.findAttribute("NumberOfLongitudesInGrid");
            final Attribute nlat = g2.findAttribute("NumberOfLatitudesInGrid");

            if (nlon == null || nlon.isString()) { continue; }
            if (nlat == null || nlat.isString()) { continue; }

            ds.addCoordinateAxis(makeLonCoordAxis(ds, g2, nlon.getNumericValue().intValue(), "XDim"));
            ds.addCoordinateAxis(makeLatCoordAxis(ds, g2, nlat.getNumericValue().intValue(), "YDim"));

            for (final Group g3 : g2.getGroups()) {
                for (final Variable v : g3.getVariables()) {
                    v.addAttribute(new Attribute(_Coordinate.Axes, "lat lon"));
                }
            }
        }

        ds.finish();
    }

/**
 *
 */
    private CoordinateAxis makeLatCoordAxis(NetcdfDataset ds, Group g, int n, String dimName) {
        final double incr = 180.0 / n;
        final CoordinateAxis v = new CoordinateAxis1D(ds, g, "lat", DataType.FLOAT, dimName,
                                                                    CDM.LAT_UNITS, "latitude");
        v.setValues(n, -90.0 + 0.5 * incr, incr);
        v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));
        return v;
    }

/**
 *
 */
    private CoordinateAxis makeLonCoordAxis(NetcdfDataset ds, Group g, int n, String dimName) {
        final double incr = 360.0 / n;
        final CoordinateAxis v = new CoordinateAxis1D(ds, g, "lon", DataType.FLOAT, dimName,
                                                                    CDM.LON_UNITS, "longitude");
        v.setValues(n, -180.0 + 0.5 * incr, incr);
        v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));
        return v;
    }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
/*
 *	This software is provided by the National Aeronatics and Space
 *	Administration Goddard Institute for Space Studies (NASA GISS) for full,
 *	free and open release. It is understood by the recipient/user that NASA
 *	assumes no liability for any errors contained in the code.  Although this
 *	software is released without conditions or restrictions in its use, it is
 *	expected that appropriate credit be given to its author and to NASA GISS
 *	should the software be included by the recipient in acknowledgments or
 *	credits in other product development.
 */
package ucar.unidata.geoloc.projection;

import ucar.nc2.constants.CF;
import ucar.unidata.geoloc.*;

/**
 * Rotated-pole longitude-latitude grid.
 *
 * This is probably the same as rotated lat lon, using matrix to do rotation.
 * Follows CF convention with "north_pole lat/lon", whereas RotatedLatLon uses south pole.
 *
 * @author Robert Schmunk
 * @author jcaron
 */
public class RotatedPole extends ProjectionImpl {
  private static final double RAD_PER_DEG = Math.PI / 180.;
  private static final double DEG_PER_RAD = 1. / RAD_PER_DEG;
  private static boolean show = false;

  /*	Coordinates of north pole for rotated pole. */
  private final ProjectionPointImpl northPole;

  /*	Y-axis rotation matrix. */
  private final double[][] rotY = new double[3][3];

  /*	Z-axis rotation matrix. */
  private final double[][] rotZ = new double[3][3];

  /**
   * Default Constructor, needed for beans.
   */
  public RotatedPole() {
    this(0.0, 0.0);
  }

  /**
   * Constructor.
   * @param northPoleLat
   * @param northPoleLon
   */
  public RotatedPole(double northPoleLat, double northPoleLon) {
    super("RotatedPole", false);

    northPole = new ProjectionPointImpl(northPoleLon, northPoleLat);
    buildRotationMatrices();

    addParameter(CF.GRID_MAPPING_NAME, CF.ROTATED_LATITUDE_LONGITUDE);
    addParameter(CF.GRID_NORTH_POLE_LATITUDE, northPoleLat);
    addParameter(CF.GRID_NORTH_POLE_LONGITUDE, northPoleLon);
  }

  private void buildRotationMatrices() {

    double betaRad = 0.;
    double gammaRad = 0.;

    if (northPole.getY() == 90.)
    {
        betaRad  = 0.;
        gammaRad = northPole.getX() * RAD_PER_DEG;
    }
    else
    {
        betaRad  = -(90. - northPole.getY()) * RAD_PER_DEG;
        gammaRad = (northPole.getX() + 180.) * RAD_PER_DEG;
    }

    double cosBeta = Math.cos(betaRad);
    double sinBeta = Math.sin(betaRad);

    double cosGamma = Math.cos(gammaRad);
    double sinGamma = Math.sin(gammaRad);

    rotY[0][0] = cosBeta;
    rotY[0][1] = 0.;
    rotY[0][2] = -sinBeta;
    rotY[1][0] = 0.;
    rotY[1][1] = 1.;
    rotY[1][2] = 0.;
    rotY[2][0] = sinBeta;
    rotY[2][1] = 0.;
    rotY[2][2] = cosBeta;

    rotZ[0][0] = cosGamma;
    rotZ[0][1] = sinGamma;
    rotZ[0][2] = 0.;
    rotZ[1][0] = -sinGamma;
    rotZ[1][1] = cosGamma;
    rotZ[1][2] = 0.;
    rotZ[2][0] = 0.;
    rotZ[2][1] = 0.;
    rotZ[2][2] = 1.;
  }

  public ProjectionPointImpl getNorthPole() {
    return new ProjectionPointImpl(northPole);
  }

  @Override
  public ProjectionImpl constructCopy() {
    ProjectionImpl result =  new RotatedPole(northPole.getY(), northPole.getX());
    result.setDefaultMapArea(defaultMapArea);
    result.setName(name);
    return result;
  }

  public String paramsToString() {
    return " northPole= " + northPole;
  }

  /**
   * Transform a "real" longitude and latitude into the rotated longitude (X) and
   * rotated latitude (Y).
   */
  public ProjectionPoint latLonToProj(LatLonPoint latlon, ProjectionPointImpl destPoint) {
    double lat = latlon.getLatitude();
    double lon = latlon.getLongitude();

    //	Lon-lat pair to xyz coordinates on sphere with radius 1
    double[] p0 = new double[]
            {Math.cos(lat * RAD_PER_DEG) * Math.cos(lon * RAD_PER_DEG),
                    Math.cos(lat * RAD_PER_DEG) * Math.sin(lon * RAD_PER_DEG),
                    Math.sin(lat * RAD_PER_DEG)};

    //	Rotate around Z-axis
    double[] p1 = new double[]
            {rotZ[0][0] * p0[0] + rotZ[0][1] * p0[1] + rotZ[0][2] * p0[2],
                    rotZ[1][0] * p0[0] + rotZ[1][1] * p0[1] + rotZ[1][2] * p0[2],
                    rotZ[2][0] * p0[0] + rotZ[2][1] * p0[1] + rotZ[2][2] * p0[2]};

    //	Rotate around Y-axis
    double[] p2 = new double[]
            {rotY[0][0] * p1[0] + rotY[0][1] * p1[1] + rotY[0][2] * p1[2],
                    rotY[1][0] * p1[0] + rotY[1][1] * p1[1] + rotY[1][2] * p1[2],
                    rotY[2][0] * p1[0] + rotY[2][1] * p1[1] + rotY[2][2] * p1[2]};

    final double lonR = LatLonPointImpl.range180( Math.atan2(p2[1], p2[0]) * DEG_PER_RAD);
    //final double lonR = Math.atan2(p2[1], p2[0]) * DEG_PER_RAD;
    final double latR = Math.asin(p2[2]) * DEG_PER_RAD;

    if (destPoint == null)
      destPoint =  new ProjectionPointImpl(lonR, latR);
    else
      destPoint.setLocation(lonR, latR);

     if (show)
        System.out.println("LatLon= " + latlon+" proj= " + destPoint);

    return destPoint;
  }

  /**
   * Transform a rotated longitude (X) and rotated latitude (Y) into a "real"
   * longitude-latitude pair.
   */
  public LatLonPoint projToLatLon(ProjectionPoint ppt, LatLonPointImpl destPoint) {
    //	"x" and "y" input for rotated pole coords are actually a lon-lat pair
    final double lonR = LatLonPointImpl.range180( ppt.getX()); // LOOK guessing
    final double latR = ppt.getY();

    //	Lon-lat pair to xyz coordinates on sphere with radius 1
    double[] p0 = new double[]
            {Math.cos(latR * RAD_PER_DEG) * Math.cos(lonR * RAD_PER_DEG),
                    Math.cos(latR * RAD_PER_DEG) * Math.sin(lonR * RAD_PER_DEG),
                    Math.sin(latR * RAD_PER_DEG)};

    //	Inverse rotate around Y-axis (using transpose of Y matrix)
    double[] p1 = new double[]
            {rotY[0][0] * p0[0] + rotY[1][0] * p0[1] + rotY[2][0] * p0[2],
                    rotY[0][1] * p0[0] + rotY[1][1] * p0[1] + rotY[2][1] * p0[2],
                    rotY[0][2] * p0[0] + rotY[1][2] * p0[1] + rotY[2][2] * p0[2]};

    //	Inverse rotate around Z-axis (using transpose of Z matrix)
    double[] p2 = new double[]
            {rotZ[0][0] * p1[0] + rotZ[1][0] * p1[1] + rotZ[2][0] * p1[2],
                    rotZ[0][1] * p1[0] + rotZ[1][1] * p1[1] + rotZ[2][1] * p1[2],
                    rotZ[0][2] * p1[0] + rotZ[1][2] * p1[1] + rotZ[2][2] * p1[2]};

    final double lon = Math.atan2(p2[1], p2[0]) * DEG_PER_RAD;
    final double lat = Math.asin(p2[2]) * DEG_PER_RAD;

    if (destPoint == null)
      destPoint = new LatLonPointImpl(lat, lon);
    else
      destPoint.set(lat, lon);

    if (show)
       System.out.println("Proj= " + ppt+" latlon= " + destPoint);

    return destPoint;
  }

  public boolean crossSeam(ProjectionPoint pt1, ProjectionPoint pt2) {
     return Math.abs(pt1.getX() - pt2.getX()) > 270.0;
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;
    RotatedPole that = (RotatedPole) o;
    return northPole.equals(that.northPole);
  }

  @Override
  public int hashCode() {
    return northPole.hashCode();
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package thredds.catalog;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import thredds.catalog.util.CatalogUtils;
import thredds.catalog.util.DeepCopyUtils;
import thredds.cataloggen.CatalogBuilder;
import thredds.cataloggen.CatalogRefExpander;
import thredds.cataloggen.DatasetEnhancer;
import thredds.cataloggen.DatasetScanCatalogBuilder;
import thredds.cataloggen.ProxyDatasetHandler;
import thredds.cataloggen.datasetenhancer.RegExpAndDurationTimeCoverageEnhancer;
import thredds.cataloggen.inserter.SimpleLatestProxyDsHandler;
import thredds.crawlabledataset.CrawlableDataset;
import thredds.crawlabledataset.CrawlableDatasetFactory;
import thredds.crawlabledataset.CrawlableDatasetFilter;
import thredds.crawlabledataset.CrawlableDatasetLabeler;
import thredds.crawlabledataset.CrawlableDatasetSorter;
import thredds.crawlabledataset.filter.RegExpMatchOnNameFilter;
import thredds.crawlabledataset.sorter.LexigraphicByNameSorter;

/**
 * Represents server-side information on how to scan a collection of datasets
 * for catalog generation.
 *
 *
Used by the THREDDS Data Server (TDS) to automatically generate catalogs.

 *
 *
Typically built from the information given by a datasetScan element in a
 * TDS config catalog.

 *
 *
Usage notes:
 *

 *
The static methods setContext() and setCatalogServletName() should only
 * be called once per web application instance. For instance, in your
 * HttpServlet implementations init() method.

 *
The method setScanLocation() should not be used; it is "public by accident".
 *

 *

 *
 *
Should be thread safe except that the above two usage notes are not enforced.
 */
public class InvDatasetScan extends InvCatalogRef {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(InvDatasetScan.class);
  static private String context = "/thredds";
  static public void setContext( String c ) { context = c; }
  static private String catalogServletName = "/catalog";
  static public void setCatalogServletName( String catServletName ) { catalogServletName = catServletName; }
  static private String makeHref(String path)
  {
    return context + (catalogServletName == null ? "" : catalogServletName) + "/" + CatalogUtils.escapePathForURL(path) + "/catalog.xml";
  }

  ////////////////////////////////////////////////
  private final String rootPath;
  private String scanLocation;

  private CrawlableDataset scanLocationCrDs;

  private final String crDsClassName;
  private final Object crDsConfigObj;

  private final CrawlableDatasetFilter filter;

  private CrawlableDatasetLabeler identifier;
  private CrawlableDatasetLabeler namer;

  private CrawlableDatasetSorter sorter;

  private Map<String, ProxyDatasetHandler> proxyDatasetHandlers;

  private boolean addDatasetSize = true;

  private List<DatasetEnhancer> childEnhancerList;
  private CatalogRefExpander catalogRefExpander;

  private boolean isValid;
  private StringBuilder invalidMessage;

  public InvDatasetScan( InvDatasetImpl parent, String name, String path, String scanLocation, String id, InvDatasetScan from )  {
    this(parent, name, path , scanLocation,
         from.crDsClassName, from.crDsConfigObj, from.filter,
         from.identifier, from.namer, from.addDatasetSize,
         from.sorter, from.proxyDatasetHandlers, from.childEnhancerList,
         from.catalogRefExpander );

    setID( id);
  }

  /**
   * Constructor. Old stuff.
   *
   * @param catalog parent catalog
   * @param parent parent dataset
   * @param name dataset name
   * @param path url path
   * @param scanLocation scan this directory
   * @param filter RegExp match on name
   * @param addDatasetSize add a size element
   * @param addLatest add a latest element
   * @param sortOrderIncreasing sort
   * @param datasetNameMatchPattern dataset naming
   * @param startTimeSubstitutionPattern time range using the file name
   * @param duration  time range using the file name
   */
  public InvDatasetScan( InvCatalogImpl catalog, InvDatasetImpl parent, String name, String path, String scanLocation, String filter,
                         boolean addDatasetSize, String addLatest, boolean sortOrderIncreasing,
                         String datasetNameMatchPattern, String startTimeSubstitutionPattern, String duration ) {

    super(parent, name, makeHref(path));
    log.debug(  "InvDatasetScan(): parent="+ parent + ", name="+ name +" , path="+ path +" , scanLocation="+ scanLocation +" , filter="+ filter +" , addLatest="+ addLatest +" , sortOrderIncreasing="+ sortOrderIncreasing +" , datasetNameMatchPattern="+ datasetNameMatchPattern +" , startTimeSubstitutionPattern= "+ startTimeSubstitutionPattern +", duration="+duration);

    this.rootPath = path;
    this.scanLocation = scanLocation;
    this.crDsClassName = null;
    this.crDsConfigObj = null;
    this.scanLocationCrDs = createScanLocationCrDs();
    this.isValid = true;
    if ( this.scanLocationCrDs == null )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=").append( path)
              .append( "; scanLocation=" ).append( scanLocation )
              .append(">: could not create CrawlableDataset for scanLocation." );
    }
    else if ( ! this.scanLocationCrDs.exists() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=").append( scanLocation )
              .append(">: CrawlableDataset for scanLocation does not exist." );
    }
    else if ( ! this.scanLocationCrDs.isCollection() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=" ).append( scanLocation )
              .append( ">: CrawlableDataset for scanLocation not a collection." );
    }

    if ( filter != null ) {
      // Include atomic datasets that match the given filter string.
      this.filter = new RegExpMatchOnNameFilter( filter);
    } else
      this.filter = null;

    this.identifier = null;
    this.namer = null;
    this.addDatasetSize = addDatasetSize;

    this.sorter = new LexigraphicByNameSorter( sortOrderIncreasing );

    // Add latest resolver dataset to Map of ProxyDatasetHandlers
    this.proxyDatasetHandlers = new HashMap<>();
    if ( addLatest != null )
    {
      if ( addLatest.equalsIgnoreCase( "true" ) )
      {
        InvService service = catalog.findService( "latest" );
        if ( service != null )
        {
          ProxyDatasetHandler proxyDsHandler = new SimpleLatestProxyDsHandler( "latest.xml", true, service, true );
          this.proxyDatasetHandlers.put( "latest.xml", proxyDsHandler );
        }
      }
    }

    if ( datasetNameMatchPattern != null
         && startTimeSubstitutionPattern != null
         && duration != null )
    {
      childEnhancerList = new ArrayList<DatasetEnhancer>();
      childEnhancerList.add(
              RegExpAndDurationTimeCoverageEnhancer
                      .getInstanceToMatchOnDatasetName( datasetNameMatchPattern,
                                                        startTimeSubstitutionPattern,
                                                        duration ) );
    }

    // catalogRefExpander = new BooleanCatalogRefExpander( this.datasetScan.??? );
  }

  /**
   * Constructor. Used by InvDatasetFeatureCollection
   *
   * @param catalog parent catalog
   * @param parent parent dataset
   * @param name dataset name
   * @param path url path
   * @param scanLocation scan this directory
   * @param filter CrawlableDatasetFilter, may be null
   * @param addDatasetSize add a size element
   * @param addLatest add a latest element
   * @param sortOrderIncreasing sort
   * @param datasetNameMatchPattern dataset naming
   * @param startTimeSubstitutionPattern time range using the file name
   * @param duration  time range using the file name
   */
  public InvDatasetScan( InvCatalogImpl catalog, InvDatasetImpl parent, String name, String path, String scanLocation, CrawlableDatasetFilter filter,
                         boolean addDatasetSize, String addLatest, boolean sortOrderIncreasing,
                         String datasetNameMatchPattern, String startTimeSubstitutionPattern, String duration ) {

    super(parent, name, makeHref(path));
    log.debug(  "InvDatasetScan(): parent="+ parent + ", name="+ name +" , path="+ path +" , scanLocation="+ scanLocation +" , filter="+ filter +" , addLatest="+ addLatest +" , sortOrderIncreasing="+ sortOrderIncreasing +" , datasetNameMatchPattern="+ datasetNameMatchPattern +" , startTimeSubstitutionPattern= "+ startTimeSubstitutionPattern +", duration="+duration);

    this.rootPath = path;
    this.scanLocation = scanLocation;
    this.filter = filter;
    this.crDsClassName = null;
    this.crDsConfigObj = null;
    this.scanLocationCrDs = createScanLocationCrDs();
    this.isValid = true;
    if ( this.scanLocationCrDs == null )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=").append( path)
              .append( "; scanLocation=" ).append( scanLocation )
              .append(">: could not create CrawlableDataset for scanLocation." );
    }
    else if ( ! this.scanLocationCrDs.exists() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=").append( scanLocation )
              .append(">: CrawlableDataset for scanLocation does not exist." );
    }
    else if ( ! this.scanLocationCrDs.isCollection() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=" ).append( scanLocation )
              .append( ">: CrawlableDataset for scanLocation not a collection." );
    }

    this.identifier = null;
    this.namer = null;
    this.addDatasetSize = addDatasetSize;

    this.sorter = new LexigraphicByNameSorter( sortOrderIncreasing );

    // Add latest resolver dataset to Map of ProxyDatasetHandlers
    this.proxyDatasetHandlers = new HashMap<>();
    if ( addLatest != null )
    {
      if ( addLatest.equalsIgnoreCase( "true" ) )
      {
        InvService service = catalog.findService( "latest" );
        if ( service != null )
        {
          ProxyDatasetHandler proxyDsHandler = new SimpleLatestProxyDsHandler( "latest.xml", true, service, true );
          this.proxyDatasetHandlers.put( "latest.xml", proxyDsHandler );
        }
      }
    }

    if ( datasetNameMatchPattern != null
         && startTimeSubstitutionPattern != null
         && duration != null )
    {
      childEnhancerList = new ArrayList<DatasetEnhancer>();
      childEnhancerList.add(
              RegExpAndDurationTimeCoverageEnhancer
                      .getInstanceToMatchOnDatasetName( datasetNameMatchPattern,
                                                        startTimeSubstitutionPattern,
                                                        duration ) );
    }

    // catalogRefExpander = new BooleanCatalogRefExpander( this.datasetScan.??? );
  }

  public InvDatasetScan( InvDatasetImpl parent, String name, String path, String scanLocation,
                         String configClassName, Object configObj, CrawlableDatasetFilter filter,
                         CrawlableDatasetLabeler identifier, CrawlableDatasetLabeler namer,
                         boolean addDatasetSize,
                         CrawlableDatasetSorter sorter, Map<String, ProxyDatasetHandler> proxyDatasetHandlers,
                         List<DatasetEnhancer> childEnhancerList, CatalogRefExpander catalogRefExpander )
  {
    super( parent, name, makeHref(path) );
    this.rootPath = path;
    this.scanLocation = scanLocation;
    this.crDsClassName = configClassName;
    this.crDsConfigObj = configObj;

    this.scanLocationCrDs = createScanLocationCrDs();
    this.isValid = true;
    if ( this.scanLocationCrDs == null )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=" ).append( scanLocation )
              .append( ">: could not create CrawlableDataset for scanLocation." );
    }
    else if ( ! this.scanLocationCrDs.exists() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=" ).append( scanLocation )
              .append( ">: CrawlableDataset for scanLocation does not exist." );
    }
    else if ( ! this.scanLocationCrDs.isCollection() )
    {
      isValid = false;
      invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( path )
              .append( "; scanLocation=" ).append( scanLocation )
              .append( ">: CrawlableDataset for scanLocation not a collection." );
    }

    this.filter = filter;
    this.identifier = identifier;
    this.namer = namer;
    this.addDatasetSize = addDatasetSize;
    this.sorter = sorter;
    this.childEnhancerList = childEnhancerList;
    this.catalogRefExpander = catalogRefExpander;

    if ( proxyDatasetHandlers == null )
      this.proxyDatasetHandlers = new HashMap<>();
    else
      this.proxyDatasetHandlers = proxyDatasetHandlers;
  }

  public String getPath() { return rootPath; }
  //public void setPath( String path) { this.rootPath = path; }

  public String getScanLocation() { return scanLocation; }

  /**
   * Resets the location being scanned (DO NOT USE THIS METHOD, "public by accident").
   *
   *
Used by DataRootHandler to allow scanning an aliased directory ("content").
   *
   * @param scanLocation the scan location.
   */
  public void setScanLocation(String scanLocation )
  {
    // ToDo LOOK Instead hand InvDatasetScan (or InvCatFactory?) an algorithm for converting an aliased location.
    if ( ! scanLocation.equals( this.scanLocation))
    {
      this.isValid = true;
      this.scanLocation = scanLocation;
      this.scanLocationCrDs = createScanLocationCrDs();
      if ( this.scanLocationCrDs == null )
      {
        isValid = false;
        invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( rootPath )
                .append( "; scanLocation=" ).append( scanLocation )
                .append( ">: could not create CrawlableDataset for scanLocation." );
      }
      else if ( !this.scanLocationCrDs.exists() )
      {
        isValid = false;
        invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( rootPath )
                .append( "; scanLocation=" ).append( scanLocation )
                .append( ">: CrawlableDataset for scanLocation does not exist." );
      }
      else if ( !this.scanLocationCrDs.isCollection() )
      {
        isValid = false;
        invalidMessage = new StringBuilder( "Invalid InvDatasetScan <path=" ).append( rootPath )
                .append( "; scanLocation=" ).append( scanLocation )
                .append( ">: CrawlableDataset for scanLocation not a collection." );
      }
    }
  }

  public String getCrDsClassName() { return crDsClassName; }
  public Object getCrDsConfigObj() { return crDsConfigObj; }

  public CrawlableDatasetFilter getFilter() { return filter; }
  public CrawlableDatasetLabeler getIdentifier() { return identifier; }

  public CrawlableDatasetLabeler getNamer() { return namer; }
  public CrawlableDatasetSorter getSorter() { return sorter; }

  public Map<String, ProxyDatasetHandler> getProxyDatasetHandlers() { return proxyDatasetHandlers; }

  public boolean getAddDatasetSize() { return addDatasetSize; }
  public List<DatasetEnhancer> getChildEnhancerList() { return childEnhancerList; }

  public CatalogRefExpander getCatalogRefExpander() { return catalogRefExpander; }

  public boolean isValid() { return isValid; }
  public String getInvalidMessage() { return invalidMessage.toString(); }

  private CrawlableDataset createScanLocationCrDs() {
    try {
      // Create the CrawlableDataset for the scan location (scanLocation).
      return CrawlableDatasetFactory.createCrawlableDataset(scanLocation, crDsClassName, crDsConfigObj);
    } catch (IllegalAccessException | NoSuchMethodException | IOException | InvocationTargetException |
            InstantiationException | ClassNotFoundException e) {
      String message = String.format(
              "createScanLocationCrDs(): failed to create CrawlableDataset for collectionLevel <%s> and class <%s>",
              scanLocation, crDsClassName);
      log.error(message, e);
      return null;
    }
  }

  private CatalogBuilder buildCatalogBuilder()
  {
    // Setup and create catalog builder.

    InvService service = getServiceDefault();
    DatasetScanCatalogBuilder dsScanCatBuilder;
    try
    {
      dsScanCatBuilder = new DatasetScanCatalogBuilder( this, scanLocationCrDs, service );
    }
    catch ( IllegalArgumentException e )
    {
      log.error( "buildCatalogBuilder(): failed to create CatalogBuilder for this collection <" + scanLocationCrDs.getPath() + ">: " + e.getMessage() );
      return null;
    }
    return dsScanCatBuilder;
  }

  /**
   * Return the CrawlableDataset path/location that corresponds to the
   * given dataset path. The given dataset path must start with the
   * datasetScan path for this InvDatasetScan, if not, a null is returned.
   *
   * @param dsPath a datasetScan dsPath that
   * @return the CrawlableDataset path that corresponds to the given dataset path or null.
   */
  public String translatePathToLocation( String dsPath )
  {
    if ( dsPath == null ) return null;

    if ( dsPath.length() > 0 )
      if ( dsPath.startsWith( "/" ) )
        dsPath = dsPath.substring( 1 );

    if ( ! dsPath.startsWith( this.getPath()))
      return null;

    // remove the matching part, the rest is the "data directory"
    String dataDir = dsPath.substring( this.getPath().length() );
    if ( dataDir.startsWith( "/" ) )
      dataDir = dataDir.substring( 1 );

    CrawlableDataset curCrDs = scanLocationCrDs.getDescendant( dataDir);

    if ( log.isDebugEnabled() ) log.debug( "translatePathToLocation(): url dsPath= " + dsPath + " to dataset dsPath= " + curCrDs.getPath() );

    return curCrDs.getPath();
  }

  /**
   * Return the CrawlableDataset for the given path, null if this InvDatasetScan
   * does not allow (filters out) the requested CrawlableDataset.
   *
   *
This method can handle requests for regular datasets and proxy datasets.
   *
   * @param path the path of the requested CrawlableDataset
   * @return the CrawlableDataset for the given path or null if the path is not allowed by this InvDatasetScan.
   * @throws IOException if an I/O error occurs while locating the children datasets.
   * @throws IllegalArgumentException if the given path is not a descendant of (or the same as) this InvDatasetScan collection level.
   */
  public CrawlableDataset requestCrawlableDataset( String path )
          throws IOException
  {
    String crDsPath = translatePathToLocation( path );
    if ( crDsPath == null )
      return null;

    CatalogBuilder catBuilder = buildCatalogBuilder();
    if ( catBuilder == null ) return null;
    return catBuilder.requestCrawlableDataset( crDsPath );
  }

  /**
   * Try to build a catalog for the given path by scanning the location
   * associated with this InvDatasetScan. The given path must start with
   * the path of this InvDatasetScan.
   *
   * @param orgPath the part of the baseURI that is the path
   * @param catURI the base URL for the catalog, used to resolve relative URLs.
   *
   * @return the catalog for this path (uses version 1.1) or null if build unsuccessful.
   */
  public InvCatalogImpl makeCatalogForDirectory( String orgPath, URI catURI ) {

    if ( log.isDebugEnabled())
    {
      log.debug( "baseURI=" + catURI );
      log.debug( "orgPath=" + orgPath );
      log.debug( "rootPath=" + rootPath );
      log.debug( "scanLocation=" + scanLocation );
    }

    // Get the dataset path.
    String dsDirPath = translatePathToLocation( orgPath );
    if ( dsDirPath == null )
    {
      String tmpMsg = "makeCatalogForDirectory(): Requesting path <" + orgPath + "> must start with \"" + rootPath + "\".";
      log.error( tmpMsg );
      return null;
    }

    // Setup and create catalog builder.
    CatalogBuilder catBuilder = buildCatalogBuilder();
    if ( catBuilder == null )
      return null;

    // A very round about way to remove the filename (e.g., "catalog.xml").
    // Note: Gets around "path separator at end of path" issues that are CrDs implementation dependant.
    // Note: Does not check that CrDs is allowed by filters.
    String dsPath = dsDirPath.substring( scanLocationCrDs.getPath().length() );
    if ( dsPath.startsWith( "/" ))
      dsPath = dsPath.substring( 1 );
    CrawlableDataset reqCrDs = scanLocationCrDs.getDescendant( dsPath );
    CrawlableDataset parent = reqCrDs.getParentDataset();
    if (parent == null) {
      log.error( "makeCatalogForDirectory(): I/O error getting parent crDs level <" + dsDirPath + ">: ");
      return null;
    }
    dsDirPath = parent.getPath();

    // Get the CrawlableDataset for the desired catalog level (checks that allowed by filters).
    CrawlableDataset catalogCrDs;
    try
    {
      catalogCrDs = catBuilder.requestCrawlableDataset( dsDirPath );
    }
    catch ( IOException e )
    {
      log.error( "makeCatalogForDirectory(): I/O error getting catalog level <" + dsDirPath + ">: " + e.getMessage(), e );
      return null;
    }

    if ( catalogCrDs == null )
    {
      log.warn( "makeCatalogForDirectory(): requested catalog level <" + dsDirPath + "> not allowed (filtered out).");
      return null;
    }
    if ( ! catalogCrDs.isCollection() )
    {
      log.warn( "makeCatalogForDirectory(): requested catalog level <" + dsDirPath + "> is not a collection.");
      return null;
    }

    // Generate the desired catalog using the builder.
    InvCatalogImpl catalog;
    try
    {
      catalog = catBuilder.generateCatalog( catalogCrDs );
    }
    catch ( IOException e )
    {
      log.error( "makeCatalogForDirectory(): catalog generation failed <" + catalogCrDs.getPath() + ">: " + e.getMessage() );
      return null;
    }

    // Set the catalog base URI.
    if ( catalog != null )
      catalog.setBaseURI( catURI );

//    InvDatasetImpl top = (InvDatasetImpl) catalog.getDataset();
//    // if we name it carefully, can get catalogRef to useProxy == true (disappear top dataset)
//    if ( service.isRelativeBase()) {
//      pos = dataDir.lastIndexOf("/");
//      String lastDir = (pos > 0) ? dataDir.substring(pos+1) : dataDir;
//      String topName = lastDir.length() > 0 ? lastDir : getName();
//      top.setName( topName);
//    }

    return catalog;
  }

  /**
   * Try to build a catalog for the given resolver path by scanning the
   * location associated with this InvDatasetScan. The given path must start
   * with the path of this InvDatasetScan and refer to a resolver
   * ProxyDatasetHandler that is part of this InvDatasetScan.
   *
   * @param path the part of the baseURI that is the path
   * @param baseURI the base URL for the catalog, used to resolve relative URLs.
   *
   * @return the resolver catalog for this path (uses version 1.1) or null if build unsuccessful.
   */
  public InvCatalogImpl makeProxyDsResolverCatalog( String path, URI baseURI )
  {
    if ( path == null ) return null;
    if ( path.endsWith( "/")) return null;

    // Get the dataset path.
    String dsDirPath = translatePathToLocation( path );
    if ( dsDirPath == null )
    {
      log.error( "makeProxyDsResolverCatalog(): Requesting path <" + path + "> must start with \"" + rootPath + "\"." );
      return null;
    }

    // Split into parent path and dataset name.
    int pos = dsDirPath.lastIndexOf('/');
    if ( pos == -1 )
    {
      log.error( "makeProxyDsResolverCatalog(): Requesting path <" + path + "> must contain a slash (\"/\")." );
      return null;
    }
    String dsName = dsDirPath.substring( pos + 1 );
    dsDirPath = dsDirPath.substring( 0, pos );

    // Find matching ProxyDatasetHandler.
    ProxyDatasetHandler pdh = this.getProxyDatasetHandlers().get( dsName );
    if ( pdh == null )
    {
      log.error( "makeProxyDsResolverCatalog(): No matching proxy dataset handler found <" + dsName + ">." );
      return null;
    }

    // Setup and create catalog builder.
    CatalogBuilder catBuilder = buildCatalogBuilder();
    if ( catBuilder == null )
      return null;

    // Get the CrawlableDataset for the desired catalog level.
    CrawlableDataset catalogCrDs;
    try
    {
      catalogCrDs = catBuilder.requestCrawlableDataset( dsDirPath );
    }
    catch ( IOException e )
    {
      log.error( "makeProxyDsResolverCatalog(): failed to create CrawlableDataset for catalogLevel <" + dsDirPath + "> and class <" + crDsClassName + ">: " + e.getMessage(), e );
      return null;
    }
    if ( catalogCrDs == null )
    {
      log.warn( "makeProxyDsResolverCatalog(): requested catalog level <" + dsDirPath + "> not allowed (filtered out)." );
      return null;
    }
    if ( ! catalogCrDs.isCollection())
    {
      log.warn( "makeProxyDsResolverCatalog(): requested catalog level <" + dsDirPath + "> not a collection." );
      return null;
    }

    // Generate the desired catalog using the builder.
    InvCatalogImpl catalog;
    try
    {
      catalog = (InvCatalogImpl) catBuilder.generateProxyDsResolverCatalog( catalogCrDs, pdh );
    }
    catch ( IOException e )
    {
      log.error( "makeProxyDsResolverCatalog(): catalog generation failed <" + catalogCrDs.getPath() + ">: " + e.getMessage() );
      return null;
    }

    // Set the catalog base URI.
    if ( catalog != null )
      catalog.setBaseURI( baseURI );

    return catalog;
  }

  /**
   * Try to build a catalog for the given path by scanning the location
   * associated with this InvDatasetScan. The given path must start with
   * the path of this InvDatasetScan.
   *
   * @param orgPath the part of the baseURI that is the path
   * @param baseURI the base URL for the catalog, used to resolve relative URLs.
   *
   * @return the catalog for this path (uses version 1.1) or null if build unsuccessful.
   *
   * @deprecated  Instead use {@link #makeProxyDsResolverCatalog(String, URI) makeProxyDsResolver()} which provides more general proxy dataset handling.
   */
  public InvCatalog makeLatestCatalogForDirectory( String orgPath, URI baseURI )
 {
   InvCatalogImpl cat = this.makeCatalogForDirectory( orgPath, baseURI );
   if ( cat == null ) return null;
   InvDatasetImpl topDs = (InvDatasetImpl) cat.getDatasets().get( 0); // Assumes catalog has one top-level dataset.

   // Keep only the regular datasets (i.e., remove CatalogRef and DatasetScan).
   for ( Iterator it = topDs.getDatasets().iterator(); it.hasNext(); )
   {
     InvDatasetImpl curDs = (InvDatasetImpl) it.next();
     if ( ! ( curDs.hasAccess()) || curDs.getUrlPath().endsWith( "latest.xml"))
     {
       it.remove();
     }
   }

   if ( topDs.getDatasets().isEmpty())
   {
     return( null);
   }

   // Grab the newest dataset.
   InvDatasetImpl latestDs = (InvDatasetImpl) java.util.Collections.max( topDs.getDatasets(), new java.util.Comparator()
   {
     public int compare( Object obj1, Object obj2 )
     {
       InvDataset ds1 = (InvDataset) obj1;
       InvDataset ds2 = (InvDataset) obj2;
       return ( ds1.getName().compareTo( ds2.getName() ) );
     }
   } );
   String baseName = topDs.getName();
   String latestName = baseName.length() == 0 ? "Latest" : "Latest " + baseName;
   latestDs.setName( latestName );

   InvCatalog subsetCat = DeepCopyUtils.subsetCatalogOnDataset( cat, latestDs );

   return ( subsetCat );
 }

  // override some InvCatalogRef stuf

   public boolean isRead() { return false; }

  boolean check(StringBuilder out, boolean show) {
    boolean isValid = true;

    if (getPath() == null) {
      out.append( "**Error: DatasetScan (" ).append( getFullName() ).append( "): must have path attribute\n" );
      isValid = false;
    }

    if ( getScanLocation() == null) {
      out.append( "**Error: DatasetScan (" ).append( getFullName() ).append( "): must have dirLocation attribute\n" );
      isValid = false;
    }

    if (getServiceDefault() == null) {
      out.append( "**Error: DatasetScan (" ).append( getFullName() ).append( "): must have a default service\n" );
      isValid = false;
    }

    if (datasets.size() > 0) {
      out.append( "**Error: DatasetScan (" ).append( getFullName() ).append( "): may not have nested datasets\n" );
      isValid = false;
      this.datasets = null;
    }

    return isValid && super.check(out, show);
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ncml;

import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.VariableDS;
import ucar.nc2.dataset.DatasetConstructor;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.util.CancelTask;
import ucar.nc2.*;
import ucar.ma2.DataType;

import java.io.IOException;
import java.util.List;

/**
 * JoinNew Aggregation.
 *
 * @author caron
 */
public class AggregationNew extends AggregationOuterDimension {

  public AggregationNew(NetcdfDataset ncd, String dimName, String recheckS) {
    super(ncd, dimName, Aggregation.Type.joinNew, recheckS);
  }

  protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
    buildCoords(cancelTask);

    // open a "typical"  nested dataset and copy it to newds
    Dataset typicalDataset = getTypicalDataset();
    NetcdfFile typical = typicalDataset.acquireFile(null);
    DatasetConstructor.transferDataset(typical, ncDataset, null);

    // create aggregation dimension
    String dimName = getDimensionName();
    Dimension aggDim = new Dimension(dimName, getTotalCoords());
    ncDataset.removeDimension(null, dimName); // remove previous declaration, if any
    ncDataset.addDimension(null, aggDim);

    promoteGlobalAttributes( (DatasetOuterDimension) typicalDataset);

    List<String> aggVarNames = getAggVariableNames();

    // Look for a variable matching the new aggregation dimension
    Variable joinAggCoord = ncDataset.findVariable(dimName);

    // Not found, create the aggregation coordinate variable
    if (joinAggCoord == null) {
      DataType coordType = getCoordinateType();
      joinAggCoord = new VariableDS(ncDataset, null, null, dimName, coordType, dimName, null, null);
      ncDataset.addVariable(null, joinAggCoord);
      joinAggCoord.setProxyReader(this);
      if (isDate)
        joinAggCoord.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, "Time"));

      // if speced externally, this variable will get replaced
      CacheVar cv = new CoordValueVar(joinAggCoord.getFullName(), joinAggCoord.getDataType(), joinAggCoord.getUnitsString());
      joinAggCoord.setSPobject(cv);
      cacheList.add(cv);
    } else if (joinAggCoord.isScalar()) {
      // For an existing variable matching the aggregated dim name, if it's a scalar
      // variable, we can just use it and its values for the aggregation coordinate variable
      // Need to ensure it's included in the list of variables to aggregate
      if (!aggVarNames.contains(joinAggCoord.getShortName())) {
        aggVarNames.add(joinAggCoord.getShortName());
      }
    } else {
      throw new IllegalArgumentException("Variable " + dimName + " already exists, but is not a scalar (suitable for aggregating as a coordinate).");
    }

    // if no names specified, add all "non-coordinate" variables.
    // Note that we haven't identified coordinate systems with CoordSysBuilder, so that info is not available.
    // So this isn't that general of a solution. But probably better than nothing
    if (aggVarNames.size() == 0) {
      for (Variable v : typical.getVariables()) {
        if (!(v instanceof CoordinateAxis))
          aggVarNames.add(v.getShortName());
      }
    }

    // now we can create all the aggNew variables
    // use only named variables
    for (String varname : aggVarNames) {
      Variable aggVar = ncDataset.findVariable(varname);
      if (aggVar == null) {
        logger.error(ncDataset.getLocation() + " aggNewDimension cant find variable " + varname);
        continue;
      }

      // construct new variable, replace old one LOOK what about Structures?
      Group newGroup =  DatasetConstructor.findGroup(ncDataset, aggVar.getParentGroup());
      VariableDS vagg = new VariableDS(ncDataset, newGroup, null, aggVar.getShortName(), aggVar.getDataType(),
          dimName + " " + aggVar.getDimensionsString(), null, null);
      vagg.setProxyReader( this);
      DatasetConstructor.transferVariableAttributes(aggVar, vagg);

      // _CoordinateAxes if it exists must be modified
      Attribute att = vagg.findAttribute(_Coordinate.Axes);
      if (att != null) {
        String axes = dimName + " " + att.getStringValue();
        vagg.addAttribute(new Attribute(_Coordinate.Axes, axes));
      }

      newGroup.removeVariable( aggVar.getShortName());
      newGroup.addVariable( vagg);
      aggVars.add(vagg);

      if (cancelTask != null && cancelTask.isCancel()) return;
    }

    setDatasetAcquireProxy(typicalDataset, ncDataset);
    typicalDataset.close( typical); // close it because we use DatasetProxyReader to acquire

    if (isDate && timeUnitsChange) {
      readTimeCoordinates(ncDataset.findVariable(dimName), cancelTask);
    }

    ncDataset.finish();
  }

  /**
   * What is the data type of the aggregation coordinate ?
   *
   * @return the data type of the aggregation coordinate
   */
  private DataType getCoordinateType() {
    List<Dataset> nestedDatasets = getDatasets();
    DatasetOuterDimension first = (DatasetOuterDimension) nestedDatasets.get(0);
    return first.isStringValued ? DataType.STRING : DataType.DOUBLE;
  }

}