/*
 * Copyright (c) 2010 The University of Reading
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University of Reading, nor the names of the
 *    authors or contributors may be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package uk.ac.rdg.resc.edal.time;

import org.joda.time.Chronology;
import org.joda.time.DateTimeField;
import org.joda.time.DurationField;

/**
 *
A {@link Chronology} in which each year has the same number of days but
 * the lengths of the months may be different.  May be useful for climate and
 * palaeoclimate calendars.

 *
In this Chronology, a millisecond instant of zero corresponds with
 * 1970-01-01T00:00:00.000Z and a year has a fixed number of milliseconds.

 *
There is no concept of an era in this calendar, so all durations and fields
 * relating to this concept are not supported.  Additionally, the concept of a
 * "weekyear" (the year that "owns" a given week) is not implemented.

 *
Instances of this class can only be created in {@link "DateTimeZone#UTC"}.
 * (Support for time zones makes little sense in this chronology).

 *
Instances of this class are immutable.

 * @author Jon Blower
 * @see "http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.4/cf-conventions.html#calendar"
 * @author Jon
 */
public class FixedYearVariableMonthChronology extends FixedYearLengthChronology {

    private final int[] monthLengths;

    private final DateTimeField dayOfMonth;
    private final DateTimeField monthOfYear;
    private final DurationField monthDuration;

    /**
     * @param monthLengths The number of days in each month
     * @throws NullPointerException if monthLengths is null
     * @throws IllegalArgumentException if monthLengths is empty or contains
     * zero or negative values.
     */
    public FixedYearVariableMonthChronology(int[] monthLengths) {
        super(sumArray(monthLengths));
        this.monthLengths = monthLengths.clone(); // defensive copy
        this.dayOfMonth = new DayOfMonthOfFixedYearDateTimeField(this);
        this.monthOfYear = new MonthOfFixedYearDateTimeField(this);
        this.monthDuration = this.monthOfYear.getDurationField();
    }

    /**
     * Calculates the sum of the values in the given array.
     * @throws NullPointerException if arr is null
     * @throws IllegalArgumentException if arr is empty or contains
     * zero or negative values.
     */
    private static int sumArray(int[] arr) {
        if (arr == null) throw new NullPointerException("null array");
        if (arr.length == 0) throw new IllegalArgumentException("Zero-length array");
        int sum = 0;
        for (int i = 0; i < arr.length; i++) {
            if (arr[i] <= 0) {
                throw new IllegalArgumentException("All array values must be > 0");
            }
            sum += arr[i];
        }
        return sum;
    }

    @Override
    public final DateTimeField dayOfMonth() {
        return this.dayOfMonth;
    }

    @Override
    public final DateTimeField monthOfYear() {
        return this.monthOfYear;
    }

    @Override
    public final DurationField months() {
        return this.monthDuration;
    }

    /**
     * Gets the length of each month in days.  Returns a new array with each
     * invocation to maintain integrity of internal data.
     * @todo return an immutable List
     */
    public int[] getMonthLengths() {
        return this.monthLengths.clone(); // Defensive copy
    }

    /** Gets the average number of milliseconds in each month */
    public long getAverageMillisInMonth() {
        return this.years().getUnitMillis() / this.monthLengths.length;
    }

    @Override
    public String toString() {
        return "Custom calendar: fixed year length, variable month length";
    }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.ui.gis;

import java.awt.geom.*;

/** Fast implementation for tracking data overlap and closest point in a 2D region.
 * The region of interest is divided into non-overlapping cells. Each cell may contain
 * zero or one data objects. This allows quickly finding if a data object may be drawn
 * (markIfClear) and closest drawn object to a point (findClosest).
 *
 * @author caron, with design help from russ
 */

public class SpatialGrid {
  private static final double MAX_DOUBLE = Double.MAX_VALUE;

  private GridCell[] gridArray[];
  private int nx, ny;
  private int countX, countY;
  private double gridWidth, gridHeight;
  private double offsetX, offsetY;

  private double scaleOverlap = 1.0;
  private Rectangle2D result = new Rectangle2D.Double();

  private boolean debug = false, debugMark = false, debugClosest = false;

  /** Constructor
   * @param nx maximum number of grid cells in x
   * @param ny : maximum number of grid cells in y
   */
  public SpatialGrid(int nx, int ny) {
    this.nx = nx;
    this.ny = ny;
    gridArray = new GridCell[ny][];
    for (int y=0; y<ny; y++) {
      gridArray[y] = new GridCell[nx];
      for (int x=0; x<nx; x++)
        gridArray[y][x] = new GridCell();
    }

  }
  /** Set the grid scale.
   * @param bbox bounding box: we are only interested in points inside of this
   * @param width : divide the bounding box into cells of this width.
   * @param height : divide the bounding box into cells of this height.
   *   maximum number of grid cells is nx x ny
   */
  public void setGrid( Rectangle2D bbox, double width, double height) {
    offsetX = bbox.getX();
    offsetY = bbox.getY();

      // number of grid cells
    countX = Math.min( nx, (int) (bbox.getWidth() / (scaleOverlap * width)));
    countY = Math.min( ny, (int) (bbox.getHeight() / (scaleOverlap * height)));

    gridWidth = bbox.getWidth() / countX;
    gridHeight = bbox.getHeight() / countY;

    if (debug)
      System.out.println("SpatialGrid size "+ gridWidth+" "+ gridHeight+" = "+countX+" by "+countY+
        " scaleOverlap= "+ scaleOverlap);
  }

  /** Set how much the data may overlap.
   * @param overlap : percent overlap
   */
  public void setOverlap(int overlap) {
    // overlap limited to [0, 50%]
    double dover = Math.max( 0.0, Math.min(.01*overlap, .50));
    scaleOverlap = 1.0 - dover;
  }

    /** clear all the grid cells  */
  public void clear() {
   for (int y=0; y<countY; y++)
     for (int x=0; x<countX; x++)
       gridArray[y][x].used = false;
  }

   /** Check if the given rect intersects an already drawn one.
    * If not, set the corresponding cell as marked, store object, return true,
    * meaning "ok to draw".
    * @param rect the bounding box of the thing we want to draw
    * @param o store this object
    * @return true if inside the bounding box and no intersection
    */
  public boolean markIfClear (Rectangle2D rect, Object o) {
    double centerX = rect.getX() + rect.getWidth()/2;
    double centerY = rect.getY() + rect.getHeight()/2;

    int indexX = (int) ((centerX-offsetX)/gridWidth);
    int indexY = (int) ((centerY-offsetY)/gridHeight);

    if (debugMark)
      System.out.println("markIfClear "+ rect+ " "+indexX+" "+indexY);

    if ((indexX < 0) || (indexX >= countX) || (indexY < 0) || (indexY >= countY)) // outside box
      return false;

    GridCell gwant = gridArray[indexY][indexX];
    if (gwant.used)  // already taken
      return false;

    if (null != findIntersection(rect))
      return false;

    // its ok to use
    gwant.used = true;
    gwant.objectBB = rect;
    gwant.o = o;
    return true;
  }

  /** Check if the given rect intersects an already drawn object
    * @param rect the bounding box of the thing we want to draw
    * @return object that intersects, or null if no intersection
    */
  public Object findIntersection (Rectangle2D rect) {
    double centerX = rect.getX() + rect.getWidth()/2;
    double centerY = rect.getY() + rect.getHeight()/2;

    int indexX = (int) ((centerX-offsetX)/gridWidth);
    int indexY = (int) ((centerY-offsetY)/gridHeight);

    // outside box
    if ((indexX < 0) || (indexX >= countX) || (indexY < 0) || (indexY >= countY))
      return null;

    // check the surrounding points
    for (int y=Math.max(0,indexY-1); y<=Math.min(countY-1,indexY+1); y++) {
      for (int x=Math.max(0,indexX-1); x<=Math.min(countX-1,indexX+1); x++) {
        GridCell gtest = gridArray[y][x];

        if (!gtest.used)
          continue;

        if (intersectsOverlap( rect, gtest.objectBB)) // hits an adjacent rectangle
          return gtest.o;
      }
    }

    return null;
  }

  /** Check if the given point is contained in already drawn object
    * @param p the point to check
    * @return object that intersects, or null if no intersection
    */
  public Object findIntersection (Point2D p) {
    int indexX = (int) ((p.getX()-offsetX)/gridWidth);
    int indexY = (int) ((p.getY()-offsetY)/gridHeight);

    // outside box
    if ((indexX < 0) || (indexX >= countX) || (indexY < 0) || (indexY >= countY))
      return null;

    // check the surrounding points
    for (int y=Math.max(0,indexY-1); y<=Math.min(countY-1,indexY+1); y++) {
      for (int x=Math.max(0,indexX-1); x<=Math.min(countX-1,indexX+1); x++) {
        GridCell gtest = gridArray[y][x];

        if (!gtest.used)
          continue;

        if (gtest.objectBB.contains( p.getX(), p.getY()))
          return gtest.o;
      }
    }

    return null;
  }

  /** Find the closest marked cell to the given point
   * @param pt  find the closest marked cell to this point
   * @return the object associated with the closest cell, or null if none
  */
  public Object findClosest(Point2D pt) {
    Object o = null;
    int indexX = (int) ((pt.getX()-offsetX)/gridWidth);
    int indexY = (int) ((pt.getY()-offsetY)/gridHeight);

    if (debugClosest)
      System.out.println("findClosest "+ pt+ " "+indexX+" "+indexY);

    if ((indexX < 0) || (indexX >= countX) || (indexY < 0) || (indexY >= countY)) // outside box
      return null;

    GridCell gwant = gridArray[indexY][indexX];
    if (gwant.used)  // that was easy
      return gwant.o;

    // check the surrounding points along perimeter of increasing diameter
    for (int p=1; p<Math.max(countX-1, countY-1); p++)
      if (null != (o = findClosestAlongPerimeter(pt, indexX, indexY, p)))
        return o;

    return null; // nothing found
  }

    // search for closest marked cell along the perimeter of square of cells
    // with center cell[x,y] and side of length 2*perimeter+1
  private Object findClosestAlongPerimeter(Point2D pt, int centerX, int centerY, int perimeter) {
    Object closestO = null;
    double closestD = MAX_DOUBLE;

    // top and bottom row
    for (int y=centerY-perimeter; y<=centerY+perimeter; y+=2*perimeter)
      for (int x=centerX-perimeter; x<=centerX+perimeter; x++) {
        double distance = distanceSq(pt,x,y);
        if (distance < closestD) {
          closestO = gridArray[y][x].o;
          closestD = distance;
          if (debugClosest) System.out.println("   closest "+ gridArray[y][x]);
        }
      }

    // middle rows
    for (int y=centerY-perimeter+1; y<=centerY+perimeter-1; y++)
      for (int x=centerX-perimeter; x<=centerX+perimeter; x+=2*perimeter) {
        double distance = distanceSq(pt,x,y);
        if (distance < closestD) {
          closestO = gridArray[y][x].o;
          closestD = distance;
          if (debugClosest) System.out.println("   closest "+ gridArray[y][x]);
        }
      }

    return closestO;
  }

    // return distance**2 from pt to center of marked cell[x,y]
    // if out of bbox or cell not marked, return MAX_DOUBLE
  private double distanceSq( Point2D pt, int indexX, int indexY) {
    if ((indexX < 0) || (indexX >= countX) || (indexY < 0) || (indexY >= countY)) // outside bounding box
      return MAX_DOUBLE;

      GridCell gtest = gridArray[indexY][indexX];
      if (!gtest.used)    // nothing in this cell
        return MAX_DOUBLE;

        // get distance from center of cell
      Rectangle2D rect = gtest.objectBB;
      double dx = rect.getX()+rect.getWidth()/2 - pt.getX();
      double dy = rect.getY()+rect.getHeight()/2 - pt.getY();
      return (dx*dx + dy*dy);
    }

  private boolean intersectsOverlap( Rectangle2D r1, Rectangle2D r2) {
    if (scaleOverlap >= 1.0)
      return r1.intersects(r2);

    Rectangle2D.intersect( r1, r2, result);
    double area = result.getWidth() * result.getHeight();
    return (area > 0) && (area > (1.0 - scaleOverlap) * r1.getWidth() * r1.getHeight());
  }

    // inner class
  private static class GridCell {
    boolean used = false;
    Rectangle2D objectBB = null;
    Object o = null;
  }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
// $Id: MultiTrajectoryObsDataset.java 63 2006-07-12 21:50:51Z edavis $
package ucar.nc2.dt.trajectory;

import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.dataset.VariableDS;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.StructurePseudo;
import ucar.nc2.VariableSimpleIF;
import ucar.nc2.dt.VariableSimpleSubclass;
import ucar.nc2.dt.*;
import ucar.nc2.units.DateUnit;
import ucar.nc2.units.SimpleUnit;
import ucar.unidata.geoloc.LatLonRect;

import java.io.IOException;
import java.util.*;

/**
 * Superclass for for implementations of TrajectoryObsDataset using a
 * NetcdfFile underneath that contains multiple trajectories.  The file
 * must have two coordinate variables one over time and one over the multiple
 * trajectories, the time dimension may be UNLIMITED (if time is not UNLIMITED,
 * there must be no UNLIMITED dimension). The file must also have a latitude
 * variable, a longitude variable, and an elevation variable each over the
 * time and trajectory dimension. All other trajectory variables must be on
 * the time and trajectory dimension with other dimensions as needed.
 * For instance:
 * <pre>
 * traj( traj)        - convertable to -> String
 * time( time)        - convertable to -> double
 * lat( time, traj)   - convertable to -> double
 * lon( time, traj)   - convertable to -> double
 * elev( time, traj)  - convertable to -> double
 * var1( time, traj[, dim#]*)
 * ...
 * varM( time, traj[, dim#]*)
 * </pre>
 *
 * @deprecated use ucar.nc2.ft.*
 * @author edavis
 * @since 13 May 2005 16:12 -0600
 */
public class MultiTrajectoryObsDataset
        extends TypedDatasetImpl
        implements TrajectoryObsDataset
{
  protected Dimension trajDim;
  protected Variable trajVar;
  protected Dimension timeDim;
  protected Variable timeVar;
  protected Structure recordVar = null;
  protected Variable latVar;
  protected Variable lonVar;
  protected Variable elevVar;

  protected String timeVarUnitsString;

  protected double elevVarUnitsConversionFactor;

  protected List trajectoryIds;
  protected List trajectories;
  protected HashMap trajectoriesMap;

  protected int trajectoryNumPoint;
  protected HashMap trajectoryVarsMap;

  public MultiTrajectoryObsDataset() {}
  public MultiTrajectoryObsDataset( NetcdfDataset ncfile)
  {
    super( ncfile);
  }

  /**
   * Setup needed for all MultiTrajectoryObsDatatypes.
   *
   * Units of time varible must be udunits time units.
   * Units of latitude variable must be convertible to "degrees_north" by udunits.
   * Units of longitude variable must be convertible to "degrees_east" by udunits.
   * Units of altitude variable must be convertible to "meters" by udunits.
   *
   * @throws IllegalArgumentException if units of time, latitude, longitude, or altitude variables are not as required.
   */
  public void setTrajectoryInfo( Dimension trajDim, Variable trajVar,
                                 Dimension timeDim, Variable timeVar,
                                 Variable latVar, Variable lonVar, Variable elevVar )
          throws IOException
  {
    this.trajDim = trajDim;
    this.trajVar = trajVar;
    this.timeDim = timeDim;
    this.timeVar = timeVar;
    this.latVar = latVar;
    this.lonVar = lonVar;
    this.elevVar = elevVar;

    trajectoryNumPoint = this.timeDim.getLength();
    timeVarUnitsString = this.timeVar.findAttribute( "units" ).getStringValue();

    // Check that time, lat, lon, elev units are acceptable.
    if ( DateUnit.getStandardDate( timeVarUnitsString ) == null )
      throw new IllegalArgumentException( "Units of time variable <" + timeVarUnitsString + "> not a date unit." );
    String latVarUnitsString = this.latVar.findAttribute( "units" ).getStringValue();
    if ( !SimpleUnit.isCompatible( latVarUnitsString, "degrees_north" ) )
      throw new IllegalArgumentException( "Units of lat var <" + latVarUnitsString + "> not compatible with \"degrees_north\"." );
    String lonVarUnitsString = this.lonVar.findAttribute( "units" ).getStringValue();
    if ( !SimpleUnit.isCompatible( lonVarUnitsString, "degrees_east" ) )
      throw new IllegalArgumentException( "Units of lon var <" + lonVarUnitsString + "> not compatible with \"degrees_east\"." );
    String elevVarUnitsString = this.elevVar.findAttribute( "units" ).getStringValue();
    if ( !SimpleUnit.isCompatible( elevVarUnitsString, "meters" ) )
      throw new IllegalArgumentException( "Units of elev var <" + elevVarUnitsString + "> not compatible with \"meters\"." );

    try
    {
      elevVarUnitsConversionFactor = getMetersConversionFactor( elevVarUnitsString );
    }
    catch ( Exception e )
    {
      throw new IllegalArgumentException( "Exception on getMetersConversionFactor() for the units of elev var <" + elevVarUnitsString + ">." );
    }

    if ( this.netcdfDataset.hasUnlimitedDimension() && this.netcdfDataset.getUnlimitedDimension().equals( timeDim))
    {
      this.netcdfDataset.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      this.recordVar = (Structure) this.netcdfDataset.getRootGroup().findVariable( "record");
    } else {
      this.recordVar = new StructurePseudo( this.netcdfDataset, null, "record", timeDim);
    }

    // @todo HACK, HACK, HACK - remove once addRecordStructure() deals with ncd attribute changes.
    Variable latVarInRecVar = this.recordVar.findVariable( this.latVar.getFullNameEscaped() );
    Attribute latVarUnitsAtt = latVarInRecVar.findAttribute( "units");
    if ( latVarUnitsAtt != null && !latVarUnitsString.equals( latVarUnitsAtt.getStringValue() ) )
      latVarInRecVar.addAttribute( new Attribute( "units", latVarUnitsString ) );
    Variable lonVarInRecVar = this.recordVar.findVariable( this.lonVar.getFullNameEscaped() );
    Attribute lonVarUnitsAtt = lonVarInRecVar.findAttribute( "units" );
    if ( lonVarUnitsAtt != null && !lonVarUnitsString.equals( lonVarUnitsAtt.getStringValue() ) )
      lonVarInRecVar.addAttribute( new Attribute( "units", lonVarUnitsString ) );
    Variable elevVarInRecVar = this.recordVar.findVariable( this.elevVar.getFullNameEscaped() );
    Attribute elevVarUnitsAtt = elevVarInRecVar.findAttribute( "units" );
    if ( elevVarUnitsAtt != null && !elevVarUnitsString.equals( elevVarUnitsAtt.getStringValue() ) )
      elevVarInRecVar.addAttribute( new Attribute( "units", elevVarUnitsString ) );

    trajectoryVarsMap = new HashMap();
    for ( Iterator it = this.netcdfDataset.getRootGroup().getVariables().iterator(); it.hasNext(); )
    {
      Variable curVar = (Variable) it.next();
      if ( curVar.getRank() >= 2 &&
         //  !curVar.equals( this.trajVar) && // These two are both one dimensional arrays
         //  !curVar.equals( this.timeVar) && //
           ! curVar.equals( this.latVar) &&
           ! curVar.equals( this.lonVar) &&
           ! curVar.equals( this.elevVar) &&
           ( this.recordVar == null ? true : ! curVar.equals( this.recordVar)))
      {
        MyTypedDataVariable typedVar = new MyTypedDataVariable( new VariableDS(  null, curVar, true ) );
        dataVariables.add( typedVar);
        trajectoryVarsMap.put( typedVar.getShortName(), typedVar);
      }
    }

    Range startPointRange = null;
    Range endPointRange = null;
    try
    {
      startPointRange = new Range( 0, 0);
      endPointRange = new Range( trajectoryNumPoint-1, trajectoryNumPoint-1);
    }
    catch ( InvalidRangeException e )
    {
      IOException ioe = new IOException( "Start or end point range invalid: " + e.getMessage());
      ioe.initCause( e);
      throw( ioe);
    }
    List section0 = new ArrayList(1);
    List section1 = new ArrayList(1);
    section0.add( startPointRange);
    section1.add( endPointRange);
    Array startTimeArray;
    Array endTimeArray;
    try
    {
      startTimeArray = this.timeVar.read( section0);
      endTimeArray = this.timeVar.read( section1);
    }
    catch ( InvalidRangeException e )
    {
      IOException ioe = new IOException( "Invalid range during read of start or end point: " + e.getMessage());
      ioe.initCause( e);
      throw( ioe);
    }

    String startTimeString;
    String endTimeString;
    if ( this.timeVar.getDataType().equals( DataType.DOUBLE))
    {
      startTimeString = startTimeArray.getDouble( startTimeArray.getIndex() ) + " " + timeVarUnitsString;
      endTimeString = endTimeArray.getDouble( endTimeArray.getIndex() ) + " " + timeVarUnitsString;
    }
    else if ( this.timeVar.getDataType().equals( DataType.FLOAT))
    {
      startTimeString = startTimeArray.getFloat( startTimeArray.getIndex() ) + " " + timeVarUnitsString;
      endTimeString = endTimeArray.getFloat( endTimeArray.getIndex() ) + " " + timeVarUnitsString;
    }
    else if ( this.timeVar.getDataType().equals( DataType.INT ) )
    {
      startTimeString = startTimeArray.getInt( startTimeArray.getIndex() ) + " " + timeVarUnitsString;
      endTimeString = endTimeArray.getInt( endTimeArray.getIndex() ) + " " + timeVarUnitsString;
    }
    else
    {
      String tmpMsg = "Time var <" + this.timeVar.getFullName() + "> is not a double, float, or integer <" + timeVar.getDataType().toString() + ">.";
      //log.error( tmpMsg );
      throw new IllegalArgumentException( tmpMsg);
    }

    startDate = DateUnit.getStandardDate( startTimeString );
    endDate = DateUnit.getStandardDate( endTimeString);

    trajectoryIds = new ArrayList();
    trajectories = new ArrayList();
    trajectoriesMap = new HashMap();
    Array trajArray = this.trajVar.read();
    Index index = trajArray.getIndex();
    for ( int i = 0; i < trajArray.getSize(); i++ )
    {
      String curTrajId;
      if ( this.trajVar.getDataType().equals( DataType.STRING ) )
      {
        curTrajId = (String) trajArray.getObject( index.set( i ) );
      }
      else if ( this.trajVar.getDataType().equals( DataType.DOUBLE ) )
      {
        curTrajId = String.valueOf( trajArray.getDouble( index.set( i ) ) );
      }
      else if ( this.trajVar.getDataType().equals( DataType.FLOAT ) )
      {
        curTrajId = String.valueOf( trajArray.getFloat( index.set( i ) ) );
      }
      else if ( this.trajVar.getDataType().equals( DataType.INT ) )
      {
        curTrajId = String.valueOf( trajArray.getInt( index.set( i ) ) );
      }
      else
      {
        String tmpMsg = "Trajectory var <" + this.trajVar.getFullName() + "> is not a string, double, float, or integer <" + this.trajVar.getDataType().toString() + ">.";
        //log.error( tmpMsg );
        throw new IllegalStateException( tmpMsg );
      }

      MultiTrajectory curTraj = new MultiTrajectory( curTrajId, i, trajectoryNumPoint, startDate, endDate,
                                                     this.trajVar, this.timeVar, timeVarUnitsString,
                                                     this.latVar, this.lonVar, this.elevVar,
                                                     dataVariables, trajectoryVarsMap );
      trajectoryIds.add( curTrajId);
      trajectories.add( curTraj);
      trajectoriesMap.put( curTrajId, curTraj );
    }
  }

  protected static double getMetersConversionFactor( String unitsString ) throws Exception
{
    SimpleUnit unit = SimpleUnit.factoryWithExceptions( unitsString );
    return unit.convertTo( 1.0, SimpleUnit.meterUnit );
  }

  public boolean syncExtend()
  {
    return false;
  }

  protected void setStartDate() {}

  protected void setEndDate() {}

  protected void setBoundingBox() {}

  public List getTrajectoryIds()
  {
    return( trajectoryIds);
  }

  public List getTrajectories() // throws IOException;
  {
    return( trajectories);
  }

  public TrajectoryObsDatatype getTrajectory( String trajectoryId ) // throws IOException;
  {
    if ( trajectoryId == null ) return( null);
    return (TrajectoryObsDatatype) trajectoriesMap.get( trajectoryId);
  }

  public String getDetailInfo()
  {
    StringBuffer sbuff = new StringBuffer();
    sbuff.append( "TrajectoryObsDataset\n" );
    sbuff.append( "  adapter   = " + getClass().getName() + "\n" );
    sbuff.append( "  trajectories:" + "\n" );
    for ( Iterator it = this.getTrajectoryIds().iterator(); it.hasNext(); )
    {
      sbuff.append( "      " + (String) it.next() + "\n" );
    }
    sbuff.append( super.getDetailInfo() );

    return sbuff.toString();
  }

  private class MultiTrajectory implements TrajectoryObsDatatype
  {

    private String id;
    private int trajNum;
    private String description;
    private int numPoints;
    private Date startDate;
    private Date endDate;
    private String timeVarUnitsString;
    private Variable trajVar;
    private Variable timeVar;
    private Variable latVar;
    private Variable lonVar;
    private Variable elevVar;
    private List variables;
    private HashMap variablesMap;

    private Range trajPointRange; // Point range on trajectory variable for this trajectory.

    private MultiTrajectory( String id, int idNum, int numPoints,
                              Date startDate, Date endDate, Variable trajVar,
                              Variable timeVar, String timeVarUnitsString,
                              Variable latVar, Variable lonVar, Variable elevVar,
                              List variables, HashMap variablesMap )
    {
      this.description = null;
      this.id = id;
      this.trajNum = idNum;
      this.numPoints = numPoints;
      this.startDate = startDate;
      this.endDate = endDate;
      this.timeVarUnitsString = timeVarUnitsString;
      this.timeVar = timeVar;
      this.trajVar = trajVar;
      this.variables = variables;
      this.variablesMap = variablesMap;
      this.latVar = latVar;
      this.lonVar = lonVar;
      this.elevVar = elevVar;

      // Calculate the point range for this trajectory in the trajectory dimension.
      try
      {
        trajPointRange = new Range( this.trajNum, this.trajNum );
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( e.getMessage() );
        iae.initCause( e );
        throw( iae );
      }
    }

    public String getId()
    {
      return( id);
    }

    public String getDescription()
    {
      return( description);
    }

    public int getNumberPoints()
    {
      return( numPoints);
    }

    public List getDataVariables()
    {
      return ( variables );
    }

    public VariableSimpleIF getDataVariable( String name )
    {
      return ( (VariableSimpleIF) variablesMap.get( name ) );
    }

    public PointObsDatatype getPointObsData( int point ) throws IOException
    {
      return( new MyPointObsDatatype( point));
    }

    public Date getStartDate()
    {
      return( startDate);
    }

    public Date getEndDate()
    {
      return( endDate);
    }

    public LatLonRect getBoundingBox()
    {
      return null;
    }

    public Date getTime( int point ) throws IOException
    {
      return ( DateUnit.getStandardDate( getTimeValue( point ) + " " + timeVarUnitsString ) );
    }

    public ucar.unidata.geoloc.EarthLocation getLocation( int point ) throws IOException
    {
      return ( new MyEarthLocation( point ) );
    }

    public String getTimeUnitsIdentifier()
    {
      return( timeVarUnitsString);
    }

    public double getTimeValue( int point ) throws IOException
    {
      Array array = null;
      try
      {
        array = getTime( this.getPointRange( point ) );
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( "Point <" + point + "> not in valid range <0, " + ( this.getNumberPoints() - 1 ) + ">: " + e.getMessage() );
        iae.initCause( e );
        throw iae;
      }
      if ( array instanceof ArrayDouble)
      {
        return( array.getDouble( array.getIndex()));
      }
      else if ( array instanceof ArrayFloat)
      {
        return( array.getFloat( array.getIndex()));
      }
      else if ( array instanceof ArrayInt )
      {
        return ( array.getInt( array.getIndex() ) );
      }
      else
      {
        throw new IOException( "Time variable not float, double, or integer <" + array.getElementType().toString() + ">.");
      }
    }

    // @todo Make sure units are degrees_north
    public double getLatitude( int point ) throws IOException // required, units degrees_north
    {
      Array array = null;
      try
      {
        array = getLatitude( this.getPointRange( point));
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( "Point <" + point + "> not in valid range <0, " + (this.getNumberPoints()-1) + ">: " + e.getMessage());
        iae.initCause( e);
        throw iae;
      }
      if ( array instanceof ArrayDouble)
      {
        return( array.getDouble( array.getIndex()));
      }
      else if ( array instanceof ArrayFloat)
      {
        return( array.getFloat( array.getIndex()));
      }
      else
      {
        throw new IOException( "Latitude variable not float or double <" + array.getElementType().toString() + ">.");
      }
    }

    // @todo Make sure units are degrees_east
    public double getLongitude( int point ) throws IOException // required, units degrees_east
    {
      Array array = null;
      try
      {
        array = getLongitude( this.getPointRange( point));
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( "Point <" + point + "> not in valid range <0, " + (this.getNumberPoints()-1) + ">: " + e.getMessage());
        iae.initCause( e);
        throw iae;
      }
      if ( array instanceof ArrayDouble)
      {
        return( array.getDouble( array.getIndex()));
      }
      else if ( array instanceof ArrayFloat)
      {
        return( array.getFloat( array.getIndex()));
      }
      else
      {
        throw new IOException( "Longitude variable not float or double <" + array.getElementType().toString() + ">.");
      }
    }

    // @todo Make sure units are meters
    public double getElevation( int point ) throws IOException // optional; units meters;  missing = NaN.
    {
      Array array = null;
      try
      {
        array = getElevation( this.getPointRange( point));
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( "Point <" + point + "> not in valid range <0, " + (this.getNumberPoints()-1) + ">: " + e.getMessage());
        iae.initCause( e);
        throw iae;
      }
      if ( array instanceof ArrayDouble)
      {
        return( array.getDouble( array.getIndex()));
      }
      else if ( array instanceof ArrayFloat)
      {
        return( array.getFloat( array.getIndex()));
      }
      else
      {
        throw new IOException( "Elevation variable not float or double <" + array.getElementType().toString() + ">.");
      }
    }

    public StructureData getData( int point ) throws IOException, InvalidRangeException
    {
      return MultiTrajectoryObsDataset.this.recordVar.readStructure( point );
    }

    public Array getData( int point, String parameterName ) throws IOException
    {
      try
      {
        return( getData( this.getPointRange( point), parameterName));
      }
      catch ( InvalidRangeException e )
      {
        IllegalArgumentException iae = new IllegalArgumentException( "Point <" + point + "> not in valid range <0, " + (this.getNumberPoints()-1) + ">: " + e.getMessage());
        iae.initCause( e);
        throw iae;
      }
    }

    public Range getFullRange()
    {
      Range range = null;
      try
      {
        range = new Range(0, this.getNumberPoints()-1);
      }
      catch ( InvalidRangeException e )
      {
        IllegalStateException ise = new IllegalStateException( "Full trajectory range invalid <0, " + (this.getNumberPoints()-1) + ">: " + e.getMessage());
        ise.initCause( e);
        throw( ise);
      }
      return range;
    }

    public Range getPointRange( int point ) throws InvalidRangeException
    {
      if ( point >= this.getNumberPoints()) throw new InvalidRangeException( "Point <" + point + "> not in acceptible range <0, " + (this.getNumberPoints()-1) + ">.");
      return( new Range(point, point));
    }

    public Range getRange( int start, int end, int stride ) throws InvalidRangeException
    {
      if ( end >= this.getNumberPoints()) throw new InvalidRangeException( "End point <" + end + "> not in acceptible range <0, " + (this.getNumberPoints()-1) + ">.");
      return( new Range( start, end, stride));
    }

    public Array getTime( Range range ) throws IOException, InvalidRangeException
    {
      List section = new ArrayList(1);
      section.add( range);
      return( timeVar.read( section));
    }

    // @todo Make sure units are degrees_north
    public Array getLatitude( Range range ) throws IOException, InvalidRangeException
    {
      List section = new ArrayList(2);
      section.add( range);
      section.add( trajPointRange);
      return( latVar.read( section).reduce());
    }

    // @todo Make sure units are degrees_east
    public Array getLongitude( Range range ) throws IOException, InvalidRangeException
    {
      List section = new ArrayList(2);
      section.add( range);
      section.add( trajPointRange );
      return( lonVar.read( section));
    }

    // @todo Make sure units are meters
    public Array getElevation( Range range ) throws IOException, InvalidRangeException
    {
      List section = new ArrayList(2);
      section.add( range);
      section.add( trajPointRange );
      Array a = elevVar.read( section);
      if ( elevVarUnitsConversionFactor == 1.0 ) return ( a );
      for ( IndexIterator it = a.getIndexIterator(); it.hasNext(); )
      {
        if ( elevVar.getDataType() == DataType.DOUBLE )
        {
          double val = it.getDoubleNext();
          it.setDoubleCurrent( val * elevVarUnitsConversionFactor );
        }
        else if ( elevVar.getDataType() == DataType.FLOAT )
        {
          float val = it.getFloatNext();
          it.setFloatCurrent( (float) ( val * elevVarUnitsConversionFactor ) );
        }
        else if ( elevVar.getDataType() == DataType.INT )
        {
          int val = it.getIntNext();
          it.setIntCurrent( (int) ( val * elevVarUnitsConversionFactor ) );
        }
        else if ( elevVar.getDataType() == DataType.LONG )
        {
          long val = it.getLongNext();
          it.setLongCurrent( (long) ( val * elevVarUnitsConversionFactor ) );
        }
        else
        {
          throw new IllegalStateException( "Elevation variable type <" + elevVar.getDataType().toString() + "> not double, float, int, or long." );
        }
      }
      return a;
    }

    public Array getData( Range range, String parameterName ) throws IOException, InvalidRangeException
    {
      Variable variable = netcdfDataset.getRootGroup().findVariable( parameterName );
      int varRank = variable.getRank();
      int [] varShape = variable.getShape();
      List section = new ArrayList( varRank);
      section.add( range);
      section.add( trajPointRange );
      for ( int i = 2; i < varRank; i++)
      {
        section.add( new Range( 0, varShape[i]-1 ) );
      }
      Array array = variable.read( section);
      array = array.reduce(1); // Reduce the trajectory dimension.
      if ( array.getShape()[0] == 1)
        return( array.reduce( 0)); // Reduce the time dimension, if a point was requested.
      else
        return( array);
    }

    public DataIterator getDataIterator( int bufferSize ) throws IOException
    {
      return null;
      //return new DataIterator( recordVar, bufferSize );
    }

//    private class DataIterator extends DatatypeIterator
//    {
//      protected Object makeDatatypeWithData( int recnum, StructureData sdata )
//      {
//        return new MyPointObsDatatype( recnum, sdata );
//      }
//
//      DataIterator( Structure struct, int bufferSize )
//      {
//        super( struct, bufferSize );
//      }
//    }

    // PointObsDatatype implementation used by MultiTrajectory.
    private class MyPointObsDatatype implements PointObsDatatype
    {
      private int point;
      private double time;
      private ucar.unidata.geoloc.EarthLocation earthLoc;

      private MyPointObsDatatype( int point) throws IOException
      {
        this.point = point;
        this.time = MultiTrajectory.this.getTimeValue( point);
        this.earthLoc = MultiTrajectory.this.getLocation( point);
      }

      public double getNominalTime()
      {
        return ( this.time );
      }

      public double getObservationTime()
      {
        return ( this.time );
      }

      public Date getNominalTimeAsDate() {
        String dateStr = getNominalTime() + " " + timeVarUnitsString;
        return DateUnit.getStandardDate( dateStr );
      }

      public Date getObservationTimeAsDate() {
        String dateStr = getObservationTime() + " " + timeVarUnitsString;
        return DateUnit.getStandardDate( dateStr );
      }

      public ucar.unidata.geoloc.EarthLocation getLocation()
      {
        return( this.earthLoc);
      }

      public StructureData getData() throws IOException
      {
        try {
          return( MultiTrajectory.this.getData( point));
        } catch (InvalidRangeException e) {
          throw new IllegalStateException( e.getMessage());
        }
      }
    }

    // EarthLocation implementation used by MultiTrajectory.
    private class MyEarthLocation extends ucar.unidata.geoloc.EarthLocationImpl
    {
      private double latitude;
      private double longitude;
      private double elevation;

      private MyEarthLocation( int point) throws IOException
      {
        this.latitude = MultiTrajectory.this.getLatitude( point);
        this.longitude = MultiTrajectory.this.getLongitude( point);
        this.elevation = MultiTrajectory.this.getElevation( point);
      }

      public double getLatitude()
      {
        return( this.latitude);
      }

      public double getLongitude()
      {
        return( this.longitude);
      }

      public double getAltitude()
      {
        return( this.elevation);
      }
    }
  }

  private class MyTypedDataVariable extends VariableSimpleSubclass
  {
    private int rank;
    private int[] shape;
    private MyTypedDataVariable( VariableDS v )
    {
      super( v );

      // Calculate the rank and shape of the variable, removing trajectory dimesion.
      rank = super.getRank();
      if ( timeDim != null ) rank--;
      if ( trajDim != null ) rank--;
      shape = new int[rank];
      int[] varShape = super.getShape();
      int trajDimIndex = v.findDimensionIndex( MultiTrajectoryObsDataset.this.trajDim.getShortName());
      int timeDimIndex = v.findDimensionIndex( MultiTrajectoryObsDataset.this.timeDim.getShortName());
      for ( int j = 0, i = 0; i < varShape.length; i++)
      {
        if ( i == trajDimIndex) continue;
        if ( i == timeDimIndex) continue;
        shape[ j++] = varShape[ i];
      }
    }

    public int getRank()
    {
      return( rank);
    }

    public int[] getShape()
    {
      return( shape);
    }
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.stream;

import com.google.common.escape.Escaper;
import com.google.common.net.UrlEscapers;
import ucar.httpservices.*;
import org.apache.http.Header;

import ucar.ma2.*;
import ucar.nc2.Structure;
import ucar.nc2.Variable;
import ucar.nc2.util.IO;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Formatter;

/**
 * A remote CDM dataset (extending NetcdfFile), using cdmremote protocol to communicate.
 * Similar to Opendap in that it is a remote access protocol using indexed data acess.
 * Supports full CDM / netcdf-4 data model.
 *
 * @author caron
 * @since Feb 7, 2009
 */
public class CdmRemote extends ucar.nc2.NetcdfFile {
  static public final String PROTOCOL = "cdmremote";
  static public final String SCHEME = PROTOCOL+":";

  // static private org.slf4SCHEMEj.Logger logger = org.slf4j.LoggerFactory.getLogger(CdmRemote.class);
  static private boolean showRequest = false;
  static private boolean compress = false;

  static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    showRequest = debugFlag.isSet("CdmRemote/showRequest");
  }

  static public void setAllowCompression(boolean b) {
    compress = b;
  }

  /**
   * Create the canonical form of the URL.
   * If the urlName starts with "http:", change it to start with "cdmremote:", otherwise
   * leave it alone.
   *
   * @param urlName the url string
   * @return canonical form
   */
  public static String canonicalURL(String urlName) {
    if (urlName.startsWith("http:"))
      return SCHEME + urlName.substring(5);
    return urlName;
  }

  //////////////////////////////////////////////////////

  private HTTPSession httpClient = null;  // stays open until close is called

  private final String remoteURI;

  public CdmRemote(String _remoteURI) throws IOException {
    long start = System.currentTimeMillis();

    // get http URL
    String temp = _remoteURI;
    try {
      if (temp.startsWith(SCHEME))
        temp = temp.substring(SCHEME.length());
      if (!temp.startsWith("http:"))
        temp = "http:" + temp;
    } catch (Exception e) {
      throw new IOException(e);
    }
    remoteURI = temp;
    httpClient = HTTPFactory.newSession(remoteURI);
    // get the header
    String url = remoteURI + "?req=header";
    if (showRequest) System.out.printf(" CdmRemote request %s%n", url);
    try (HTTPMethod method = HTTPFactory.Get(httpClient, url)) {
      method.setFollowRedirects(true);
      int statusCode = method.execute();

      if (statusCode == 404)
        throw new FileNotFoundException(getErrorMessage(method));

       if (statusCode >= 300)
        throw new IOException(getErrorMessage(method));

      InputStream is = method.getResponseAsStream();
      NcStreamReader reader = new NcStreamReader();
      reader.readStream(is, this);
      this.location = SCHEME + remoteURI;
    }

    long took = System.currentTimeMillis() - start;
    if (showRequest) System.out.printf(" CdmRemote request %s took %d msecs %n", url, took);
  }

  // Closes the input stream.
  public CdmRemote(InputStream is, String location ) throws IOException {
    remoteURI = location;

    try {
      NcStreamReader reader = new NcStreamReader();
      reader.readStream(is, this);
      this.location = SCHEME + remoteURI;

    } finally {
      is.close();
    }
  }

  @Override
  protected Array readData(ucar.nc2.Variable v, Section section) throws IOException, InvalidRangeException {
    //if (unlocked)
    //  throw new IllegalStateException("File is unlocked - cannot use");

    if (v.getDataType() == DataType.SEQUENCE) {
      Structure s = (Structure) v;
      StructureDataIterator siter = getStructureIterator(s, -1);
      return new ArraySequence(s.makeStructureMembers(), siter, -1);
    }

    Formatter f = new Formatter();
    f.format("%s?req=data", remoteURI);
    if (compress)
      f.format("&deflate=5");
    //f.format("&var=%s", v.getShortName());
    f.format("&var=%s", v.getFullNameEscaped());
    if ((section != null) && (section.computeSize() != v.getSize()) && (v.getDataType() != DataType.SEQUENCE)) {
      f.format("(%s)", section.toString());
    }
    String url = f.toString();
    // String escapedURI = f.toString();
    URI escapedURI;
    try {
      escapedURI = HTTPUtil.parseToURI(url);
    } catch (URISyntaxException e) {
      throw new RuntimeException(e);
    }

    if (showRequest)
      System.out.printf("CdmRemote data request for variable: '%s' section=(%s)%n url='%s'%n esc='%s'%n",
              v.getFullName(), section, url, escapedURI);

    try (HTTPMethod method = HTTPFactory.Get(httpClient, escapedURI.toString())) {
      int statusCode = method.execute();
      if (statusCode == 404)
        throw new FileNotFoundException(getErrorMessage(method));

      if (statusCode >= 300)
        throw new IOException(getErrorMessage(method));

      Header h = method.getResponseHeader("Content-Length");
      if (h != null) {
        String s = h.getValue();
        int readLen = Integer.parseInt(s);
        if (showRequest)
          System.out.printf(" content-length = %d%n", readLen);
        if (v.getDataType() != DataType.SEQUENCE) {
          int wantSize = (int) (v.getElementSize() * (section == null ? v.getSize() : section.computeSize()));
          if (readLen != wantSize)
            throw new IOException("content-length= " + readLen + " not equal expected Size= " + wantSize); // LOOK
        }
      }

      InputStream is = method.getResponseAsStream();  // Closed by HTTPMethod.close().
      NcStreamReader reader = new NcStreamReader();
      NcStreamReader.DataResult result = reader.readData(is, this, remoteURI);

      assert v.getFullNameEscaped().equals(result.varNameFullEsc);
      return result.data;
    }
  }

  private static String getErrorMessage(HTTPMethod method) {
    String path = method.getURI().toString();
    String status = method.getStatusLine();
    String content = method.getResponseAsString();
    return (content == null) ? status + " " + path : status + " " + path +"\n "+content;
  }

  protected StructureDataIterator getStructureIterator(Structure s, int bufferSize) throws java.io.IOException {
    try {
      InputStream is = sendQuery(httpClient, remoteURI, s.getFullNameEscaped());
      NcStreamReader reader = new NcStreamReader();
      return reader.getStructureIterator(is, this);

    } catch (Throwable e) {
      e.printStackTrace();
      throw new IllegalStateException(e);
    }
  }

  // session may be null, if so, will be closed on method.close()
  public static InputStream sendQuery(HTTPSession session, String remoteURI, String query) throws IOException {
    long start = System.currentTimeMillis();

    StringBuilder sbuff = new StringBuilder(remoteURI);
    sbuff.append("?");
    sbuff.append(query);
    if (showRequest) System.out.printf(" CdmRemote sendQuery= %s", sbuff);

    HTTPMethod method = HTTPFactory.Get(session, sbuff.toString());
    try {
      int statusCode = method.execute();
      if (statusCode == 404) {
        throw new FileNotFoundException(method.getPath() + " " + method.getStatusLine());
      } else if (statusCode >= 400) {
        throw new IOException(method.getPath() + " " + method.getStatusLine());
      }

      InputStream stream = method.getResponseBodyAsStream();
      if (showRequest) System.out.printf(" took %d msecs %n", System.currentTimeMillis() - start);

      // Leave the stream open. We must also leave the HTTPMethod open because the two are linked:
      // calling close() on one object causes the other object to be closed as well.
      return stream;
    } catch (IOException e) {
      // Close the HTTPMethod if there was an exception; otherwise leave it open.
      method.close();
      throw e;
    }
  }

  @Override
  public String getFileTypeId() {
    return "ncstreamRemote";
  }

  @Override
  public String getFileTypeDescription() {
    return "ncstreamRemote";
  }

  public void writeToFile(String filename) throws IOException {
    File file = new File(filename);
    String url = remoteURI + "?req=header";
    Escaper urlParamEscaper = UrlEscapers.urlFormParameterEscaper();

    try (FileOutputStream fos = new FileOutputStream(file)) {

      long size = 4;
      fos.write(NcStream.MAGIC_START);

      // header
      try (HTTPMethod method = HTTPFactory.Get(httpClient, url)) {
        if (showRequest) System.out.printf("CdmRemote request %s %n", url);
        int statusCode = method.execute();

        if (statusCode == 404)
          throw new FileNotFoundException(getErrorMessage(method));

        if (statusCode >= 300)
          throw new IOException(getErrorMessage(method));

        InputStream is = method.getResponseBodyAsStream();
        size += IO.copyB(is, fos, IO.default_socket_buffersize);
      }

      for (Variable v : getVariables()) {
        StringBuilder sbuff = new StringBuilder(remoteURI);
        sbuff.append("?var=");
        sbuff.append(urlParamEscaper.escape(v.getShortName()));

        if (showRequest)
          System.out.println(" CdmRemote data request for variable: " + v.getFullName() + " url=" + sbuff);

        try (HTTPMethod method = HTTPFactory.Get(httpClient, sbuff.toString())) {
          int statusCode = method.execute();

          if (statusCode == 404)
            throw new FileNotFoundException(getErrorMessage(method));

          if (statusCode >= 300)
            throw new IOException(getErrorMessage(method));

          int wantSize = (int) (v.getSize());
          Header h = method.getResponseHeader("Content-Length");
          if (h != null) {
            String s = h.getValue();
            int readLen = Integer.parseInt(s);
            if (readLen != wantSize)
              throw new IOException("content-length= " + readLen + " not equal expected Size= " + wantSize);
          }

          InputStream is = method.getResponseBodyAsStream();
          size += IO.copyB(is, fos, IO.default_socket_buffersize);
        }
      }

      fos.flush();
    }
  }

  @Override
  public synchronized void close() throws java.io.IOException {
    if (httpClient != null) httpClient.close();
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.unidata.geoloc.vertical;

import ucar.ma2.*;
import ucar.ma2.ArrayDouble.D1;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.unidata.util.Parameter;

import java.io.IOException;
import java.util.List;

/**
 * Create a 3D height(z,y,x) array using the CF formula for
 * "ocean sigma vertical coordinate".
 *
 * @author caron
 * @see http://cf-pcmdi.llnl.gov/
 */

public class OceanSigma extends VerticalTransformImpl {

  /**
   * The eta variable name identifier
   */
  public static final String ETA = "Eta_variableName";

  /**
   * The "s" variable name identifier
   */
  public static final String SIGMA = "Sigma_variableName";

  /**
   * The "depth" variable name identifier
   */
  public static final String DEPTH = "Depth_variableName";

  /**
   * The eta, s and depth variables
   */
  private Variable etaVar, sVar, depthVar;

  /**
   * Create a new vertical transform for Ocean S coordinates
   *
   * @param ds      dataset
   * @param timeDim time dimension
   * @param params  list of transformation Parameters
   */
  public OceanSigma(NetcdfFile ds, Dimension timeDim, List<Parameter> params) {

    super(timeDim);
    String etaName = getParameterStringValue(params, ETA);
    String sName = getParameterStringValue(params, SIGMA);
    String depthName = getParameterStringValue(params, DEPTH);

    etaVar = ds.findVariable(etaName);
    sVar = ds.findVariable(sName);
    depthVar = ds.findVariable(depthName);

    units = ds.findAttValueIgnoreCase(depthVar, CDM.UNITS, "none");
  }

  /**
   * Get the 3D vertical coordinate array for this time step.
   *
   * @param timeIndex the time index. Ignored if !isTimeDependent().
   * @return vertical coordinate array
   * @throws IOException           problem reading data
   * @throws InvalidRangeException _more_
   */
  public ArrayDouble.D3 getCoordinateArray(int timeIndex)
      throws IOException, InvalidRangeException {
    Array eta = readArray(etaVar, timeIndex);
    Array sigma = readArray(sVar, timeIndex);
    Array depth = readArray(depthVar, timeIndex);

    int nz = (int) sigma.getSize();
    Index sIndex = sigma.getIndex();

    int[] shape2D = eta.getShape();
    int ny = shape2D[0];
    int nx = shape2D[1];
    Index etaIndex = eta.getIndex();
    Index depthIndex = depth.getIndex();

    ArrayDouble.D3 height = new ArrayDouble.D3(nz, ny, nx);

    for (int z = 0; z < nz; z++) {
      double sigmaVal = sigma.getDouble(sIndex.set(z));
      for (int y = 0; y < ny; y++) {
        for (int x = 0; x < nx; x++) {
          double etaVal = eta.getDouble(etaIndex.set(y, x));
          double depthVal = depth.getDouble(depthIndex.set(y, x));

          height.set(z, y, x,
              etaVal + sigmaVal * (depthVal + etaVal));
        }
      }
    }

    return height;
  }

  /**
   * Get the 1D vertical coordinate array for this time step and point
   *
   * @param timeIndex the time index. Ignored if !isTimeDependent().
   * @param xIndex    the x index
   * @param yIndex    the y index
   * @return vertical coordinate array
   * @throws java.io.IOException problem reading data
   * @throws ucar.ma2.InvalidRangeException _more_
   */
  public D1 getCoordinateArray1D(int timeIndex, int xIndex, int yIndex)
		  throws IOException, InvalidRangeException {

	  Array eta = readArray(etaVar, timeIndex);
	  Array sigma = readArray(sVar, timeIndex);
	  Array depth = readArray(depthVar, timeIndex);

	  int nz = (int) sigma.getSize();
	  Index sIndex = sigma.getIndex();

	  Index etaIndex = eta.getIndex();
	  Index depthIndex = depth.getIndex();

	  ArrayDouble.D1 height = new ArrayDouble.D1(nz);

	  for (int z = 0; z < nz; z++) {
		  double sigmaVal = sigma.getDouble(sIndex.set(z));

		  double etaVal = eta.getDouble(etaIndex.set(yIndex, xIndex));
		  double depthVal = depth.getDouble(depthIndex.set(yIndex, xIndex));

		  height.set(z, etaVal + sigmaVal * (depthVal + etaVal));
	  }

	  return height;
  }

}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.units;

import javax.annotation.concurrent.Immutable;
import java.util.SortedMap;
import java.util.TreeMap;

/**
 * Provides support for base units.
 *
 * @author Steven R. Emmerson
 */
@Immutable
public class BaseUnit extends DerivedUnitImpl implements Base {
    private static final long                              serialVersionUID = 1L;

    /**
     * The identifier-to-unit map.
     *
     * @serial
     */
    private static final SortedMap<UnitName, BaseUnit>     nameMap          = new TreeMap<>();

    /**
     * The quantity-to-unit map.
     *
     * @serial
     */
    private static final SortedMap<BaseQuantity, BaseUnit> quantityMap      = new TreeMap<>();

    /**
     * The base quantity associated with this base unit.
     *
     * @serial
     */
    private final BaseQuantity                             baseQuantity;

    /**
     * Constructs from identifiers and a base quantity.
     *
     * @param id
     *            The identifiers for the base unit.
     *				id.getSymbol()
     *            shall not return
     *				null.
     * @param baseQuantity
     *            The base quantity of the base unit.
     * @throws NameException
     *             id.getSymbol() returned
     *				null.
     */
    protected BaseUnit(final UnitName id, final BaseQuantity baseQuantity)
            throws NameException {
        super(id);
        if (id.getSymbol() == null) {
            throw new NameException("Base unit must have symbol");
        }
        setDimension(new UnitDimension(this));
        this.baseQuantity = baseQuantity;
    }

    /**
     * Factory method for creating a new BaseUnit or obtaining a
     * previously-created one.
     *
     * @param id
     *            The identifier for the base unit.
     *				id.getSymbol()
     *            shall not return
     *				null.
     * @param baseQuantity
     *            The base quantity of the base unit.
     * @throws NameException
     *             id.getSymbol() returned
     *				null.
     * @throws UnitExistsException
     *             Attempt to incompatibly redefine an existing base unit.
     */
    public static synchronized BaseUnit getOrCreate(final UnitName id,
            final BaseQuantity baseQuantity) throws NameException,
            UnitExistsException {
        BaseUnit baseUnit;
        final BaseUnit nameUnit = nameMap.get(id);
        final BaseUnit quantityUnit = quantityMap.get(baseQuantity);
        if (nameUnit != null || quantityUnit != null) {
            baseUnit = nameUnit != null
                    ? nameUnit
                    : quantityUnit;
            if ((nameUnit != null && !baseQuantity.equals(nameUnit
                    .getBaseQuantity()))
                    || (quantityUnit != null && !id.equals(quantityUnit
                            .getUnitName()))) {
                throw new UnitExistsException(
                        "Attempt to incompatibly redefine base unit \""
                                + baseUnit + '"');
            }
        }
        else {
            baseUnit = new BaseUnit(id, baseQuantity);
            quantityMap.put(baseQuantity, baseUnit);
            nameMap.put(id, baseUnit);
        }
        return baseUnit;
    }

    /**
     * Returns the base quantity associated with this base unit.
     *
     * @return The base quantity associated with this base unit.
     */
    public final BaseQuantity getBaseQuantity() {
        return baseQuantity;
    }

    /**
     * Returns the identifier for this base unit. This is identical to
     * getSymbol().
     *
     * @return The identifier for this base unit.
     */
    public final String getID() {
        return getSymbol();
    }

    /**
     * Returns the string representation of this base unit. This is identical to
     * getID().
     *
     * @return The string representation of this base unit.
     */
    @Override
    public final String toString() {
        return getID();
    }

    /**
     * Indicates if this base unit is dimensionless.
     *
     * @return true if and only if this base unit is dimensionless
     *         (e.g. "radian").
     */
    @Override
    public boolean isDimensionless() {
        return baseQuantity.isDimensionless();
    }

    /**
     * Tests this class.
     */
    public static void main(final String[] args) throws Exception {
        final BaseUnit meter = new BaseUnit(UnitName.newUnitName("meter", null,
                "m"), BaseQuantity.LENGTH);
        System.out
                .println("meter.getBaseQuantity()=" + meter.getBaseQuantity());
        System.out
                .println("meter.toDerivedUnit(1.)=" + meter.toDerivedUnit(1.));
        System.out.println("meter.toDerivedUnit(new float[] {2})[0]="
                + meter.toDerivedUnit(new float[] { 2 }, new float[1])[0]);
        System.out.println("meter.fromDerivedUnit(1.)="
                + meter.fromDerivedUnit(1.));
        System.out.println("meter.fromDerivedUnit(new float[] {3})[0]="
                + meter.fromDerivedUnit(new float[] { 3 }, new float[1])[0]);
        System.out.println("meter.isCompatible(meter)="
                + meter.isCompatible(meter));
        final BaseUnit radian = new BaseUnit(UnitName.newUnitName("radian",
                null, "rad"), BaseQuantity.PLANE_ANGLE);
        System.out.println("meter.isCompatible(radian)="
                + meter.isCompatible(radian));
        System.out
                .println("meter.isDimensionless()=" + meter.isDimensionless());
        System.out.println("radian.isDimensionless()="
                + radian.isDimensionless());
    }
}


/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package thredds.inventory;

import ucar.nc2.units.TimeDuration;
import ucar.nc2.util.CloseableIterator;
import ucar.nc2.util.ListenerManager;

import java.io.IOException;
import java.util.*;

/**
 * Abstract superclass for implementations of CollectionManager.
 *
 * @author caron
 * @since Jan 19, 2010
 */
public abstract class CollectionManagerAbstract extends CollectionAbstract implements CollectionManager  {

    // called from Aggregation, Fmrc, FeatureDatasetFactoryManager
  static public CollectionManager open(String collectionName, String collectionSpec, String olderThan, Formatter errlog) throws IOException {
    if (collectionSpec.startsWith(CATALOG))
      return new CollectionManagerCatalog(collectionName, collectionSpec, olderThan, errlog);
    else
      return MFileCollectionManager.open(collectionName, collectionSpec, olderThan, errlog);
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  protected TimeDuration recheck;
  private ListenerManager lm; // lazy init
  private boolean isStatic; // true if theres no update element. It means dont scan if index already exists

  // these actually dont change, but are not set in the constructor  now set in CollectionAbstract
  //protected DateExtractor dateExtractor;
  //protected CalendarDate startCollection;

  protected CollectionManagerAbstract( String collectionName, org.slf4j.Logger logger) {
    super(collectionName, logger);
  }

  @Override
  public boolean isStatic() {
    return isStatic;
  }

  public void setStatic(boolean aStatic) {
    isStatic = aStatic;
  }

  @Override
  public TimeDuration getRecheck() {
    return recheck;
  }

  // fake default implementation
  @Override
  public CloseableIterator<MFile> getFileIterator() throws IOException {
    return new MFileIterator( getFilesSorted().iterator(), null);
  }

  @Override
  public void close() {
    if (store != null) store.close();
  }

  @Override
  public boolean scanIfNeeded() throws IOException {
    // if (map == null && !isStatic()) return true;
    return isScanNeeded() && scan(false);
  }

  /////////////////////////////////////////////////////////////////////
  // experimental
  // use bdb to manage metadata associated with the collection. currently, only DatasetInv.xml files

  private static StoreKeyValue.Factory storeFactory;
  static public void setMetadataStore(StoreKeyValue.Factory _storeFactory) {
    storeFactory = _storeFactory;
  }

  private StoreKeyValue store;

  private void initMM() {
    if (getCollectionName() == null) return; // eg no scan in ncml
    try {
    	if (storeFactory != null)
    		store = storeFactory.open(getCollectionName());

    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e.getMessage());
    }
  }

  /* clean up deleted files in metadata manager
  protected void deleteOld(Map<String, MFile> newMap) {
    if (store == null && enableMetadataManager) initMM();
    if (store != null) store.delete(newMap);
  } */

  public void putMetadata(MFile file, String key, byte[] value) {
    if (store == null) initMM();
    if (store != null) store.put(file.getPath()+"#"+key, value);
  }

  public byte[] getMetadata(MFile file, String key) {
    if (store == null) initMM();
    return (store == null) ? null : store.getBytes(file.getPath()+"#"+key);
  }

  void sendEvent(TriggerEvent event) {
    if (lm != null)
      lm.sendEvent(event);
  }

  @Override
  public void addEventListener(TriggerListener l) {
    if (lm == null) createListenerManager();
    lm.addListener(l);
  }

  @Override
  public void removeEventListener(TriggerListener l) {
    if (lm != null)
      lm.removeListener(l);
  }

  protected void createListenerManager() {
    lm = new ListenerManager(
            "thredds.inventory.CollectionManager$TriggerListener",
            "thredds.inventory.CollectionManager$TriggerEvent",
            "handleCollectionEvent");
  }

}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.dataset;

import org.apache.http.Header;
import org.apache.http.HttpStatus;
import thredds.client.catalog.ServiceType;
import thredds.client.catalog.tools.DataFactory;
import ucar.httpservices.HTTPFactory;
import ucar.httpservices.HTTPMethod;
import ucar.nc2.stream.CdmRemote;
import ucar.nc2.util.EscapeStrings;
import ucar.unidata.util.StringUtil2;
import ucar.unidata.util.Urlencoded;

import java.io.*;
import java.util.*;

/**
 * Detection of the protocol from a location string.
 * Split out from NetcdfDataset.
 * LOOK should be refactored
 *
 * @author caron
 * @since 10/20/2015.
 */
public class DatasetUrl {
  static final protected String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
  static final protected String slashalpha = "\\/" + alpha;

  static final String[] FRAGPROTOCOLS =
          {"dap4", "dap2", "dods", "cdmremote", "thredds", "ncml"};
  static final ServiceType[] FRAGPROTOSVCTYPE =
          {ServiceType.DAP4, ServiceType.OPENDAP, ServiceType.OPENDAP, ServiceType.THREDDS, ServiceType.THREDDS, ServiceType.NCML};

  /**
   * Return the set of leading protocols for a url; may be more than one.
   * Watch out for Windows paths starting with a drive letter => protocol
   * names must all have a length > 1.
   * Watch out for '::'
   * Each captured protocol is saved without trailing ':'
   * Assume: the protocols MUST be terminated by the occurrence of '/'.
   *
   * @param url the url whose protocols to return
   * @return list of leading protocols without the trailing :
   */
  static public List<String> getProtocols(String url) {
    List<String> allprotocols = new ArrayList<>(); // all leading protocols upto path or host

    // Note, we cannot use split because of the context sensitivity
    // This code is quite ugly because of all the confounding cases
    // (e.g. windows path, embedded colons, etc.).
    // Specifically, the 'file:' protocol is a problem because
    // it has no many non-standard forms such as file:x/y file://x/y file:///x/y.
    StringBuilder buf = new StringBuilder(url);
    // If there are any leading protocols, then they must stop at the first '/'.
    int slashpos = buf.indexOf("/");
    // Check special case of file:<path> with no slashes after file:
    if (url.startsWith("file:") && "/\\".indexOf(url.charAt(5)) < 0) {
      allprotocols.add("file");
    } else if (slashpos >= 0) {
      // Remove everything after the first slash
      buf.delete(slashpos + 1, buf.length());
      for (; ; ) {
        int index = buf.indexOf(":");
        if (index < 0) break; // no more protocols
        // Validate protocol
        if (!validateprotocol(url, 0, index))
          break;
        String protocol = buf.substring(0, index);  // not including trailing ':'
        allprotocols.add(protocol);
        buf.delete(0, index + 1); // remove the leading protocol
      }
    }
    return allprotocols;
  }

  static private boolean validateprotocol(String url, int startpos, int endpos) {
    int len = endpos - startpos;
    if (len == 0) return false;
    char cs = url.charAt(startpos);
    char ce1 = url.charAt(endpos + 1);
    if (len == 1 //=>|protocol| == 1
            && alpha.indexOf(cs) >= 0 && "/\\".indexOf(ce1) >= 0)
      return false; // looks like windows drive letter
    // If trailing colon is not followed by alpha or /, then assume not url
    if (slashalpha.indexOf(ce1) < 0)
      return false;
    return true;
  }

  /////////////////////////////////////////////////////////////////////////////////////

  static public DatasetUrl findDatasetUrl(String orgLocation) throws IOException {
    ServiceType svctype = null;

    // Canonicalize the location
    String location = StringUtil2.replace(orgLocation.trim(), '\\', "/");
    List<String> allprotocols = DatasetUrl.getProtocols(location);

    String trueurl = location;
    String leadprotocol;
    if (allprotocols.size() == 0) {
      leadprotocol = "file";  // The location has no leading protocols, assume file:
    } else {
      leadprotocol = allprotocols.get(0);
    }

    // Priority in deciding
    // the service type is as follows.
    // 1. "protocol" tag in fragment
    // 2. specific protocol in fragment
    // 3. leading protocol
    // 4. path extension
    // 5. contact the server (if defined)

    // temporarily remove any trailing query or fragment
    String fragment = null;
    int pos = trueurl.lastIndexOf('#');
    if (pos >= 0) {
      fragment = trueurl.substring(pos + 1, trueurl.length());
      trueurl = trueurl.substring(0, pos);
    }
    pos = location.lastIndexOf('?');
    String query = null;
    if (pos >= 0) {
      query = trueurl.substring(pos + 1, trueurl.length());
      trueurl = trueurl.substring(0, pos);
    }
    if (fragment != null)
      svctype = searchFragment(fragment);

    if (svctype == null) // See if leading protocol tells us how to interpret
      svctype = decodeLeadProtocol(leadprotocol);

    if (svctype == null) // See if path tells us how to interpret
      svctype = searchPath(trueurl);

    if (svctype == null) {
      //There are several possibilities at this point; all of which
      // require further info to disambiguate
      //  - we have file://<path> or file:<path>; we need to see if
      //    the extension can help, otherwise, start defaulting.
      //  - we have a simple url: e.g. http://... ; contact the server
      if (leadprotocol.equals("file")) {
        svctype = decodePathExtension(trueurl); // look at the path extension
        if(svctype == null && checkIfNcml(new File(location))) {
          svctype = ServiceType.NCML;
        }
      } else {
        svctype = disambiguateHttp(trueurl);
        // special cases
        if ((svctype == null || svctype == ServiceType.HTTPServer)) {
          // ncml file being served over http?
          if (checkIfRemoteNcml(trueurl)) {
            svctype = ServiceType.NCML;
          }
        }
      }
    }

    if (svctype == ServiceType.NCML) { // ??
      // If lead protocol was null and then pretend it was a file
      // Note that technically, this should be 'file://'
      trueurl = (allprotocols.size() == 0 ? "file:" + trueurl : location);
    }

    // Add back the query and fragment (if any)
    if (query != null || fragment != null) {
      StringBuilder buf = new StringBuilder(trueurl);
      if (query != null) {
        buf.append('?');
        buf.append(query);
      }
      if (fragment != null) {
        buf.append('#');
        buf.append(fragment);
      }
      trueurl = buf.toString();
    }
    return new DatasetUrl(svctype, trueurl);
  }

  /**
   * Given a location, find markers indicated which protocol to use
   * LOOK what use case is this handling ?
   *
   * @param fragment the fragment is to be examined
   * @return The discovered ServiceType, or null
   */
  static private ServiceType searchFragment(String fragment) {
    if (fragment.length() == 0)
      return null;
    Map<String, String> map = parseFragment(fragment);
    if (map == null) return null;
    String protocol = map.get("protocol");
    if(protocol == null) {
      for(String p: FRAGPROTOCOLS) {
        if(map.get(p) != null) {protocol = p; break;}
      }
    }
    if (protocol != null) {
      if (protocol.equalsIgnoreCase("dap") || protocol.equalsIgnoreCase("dods"))
        return ServiceType.OPENDAP;
      if (protocol.equalsIgnoreCase("dap4"))
        return ServiceType.DAP4;
      if (protocol.equalsIgnoreCase("cdmremote"))
        return ServiceType.CdmRemote;
      if (protocol.equalsIgnoreCase("thredds"))
        return ServiceType.THREDDS;
      if (protocol.equalsIgnoreCase("ncml"))
        return ServiceType.NCML;
    }
    return null;
  }

  /**
   * Given the fragment part of a url, see if it
   * parses as name=value pairs separated by '&'
   * (same as query part).
   *
   * @param fragment the fragment part of a url
   * @return a map of the name value pairs (possibly empty),
   * or null if the fragment does not parse.
   */
  static private Map<String, String> parseFragment(String fragment) {
    Map<String, String> map = new HashMap<>();
    if (fragment != null && fragment.length() >= 0) {
      if (fragment.charAt(0) == '#')
        fragment = fragment.substring(1);
      String[] pairs = fragment.split("[ \t]*[&][ \t]*");
      for (String pair : pairs) {
        String[] pieces = pair.split("[ \t]*[=][ \t]*");
        switch (pieces.length) {
          case 1:
            map.put(EscapeStrings.unescapeURL(pieces[0]).toLowerCase(), "true");
            break;
          case 2:
            map.put(EscapeStrings.unescapeURL(pieces[0]).toLowerCase(),
                    EscapeStrings.unescapeURL(pieces[1]).toLowerCase());
            break;
          default:
            return null; // does not parse
        }
      }
    }
    return map;
  }

  /**
   * Given a url, search the path to look for protocol indicators
   *
   * @param url the url is to be examined
   * @return The discovered ServiceType, or null
   */
  static private ServiceType searchPath(String url) {
      if(false) { // Disable for now
      if(url == null || url.length() == 0)
        return null;
      url = url.toLowerCase(); // for matching purposes
      for(int i=0; i<FRAGPROTOCOLS.length;i++) {
        String p = FRAGPROTOCOLS[i];
        if(url.indexOf("/thredds/"+p.toLowerCase()+"/")>= 0) {
          return FRAGPROTOSVCTYPE[i];
        }
      }
      }
      return null;
    }

    /**
     * Check path extension; assumes no query or fragment
     *
     * @param path the path to examine for extension
     * @return ServiceType inferred from the extension or null
     */
  static private ServiceType decodePathExtension(String path) {
    // Look at the path extensions
    if (path.endsWith(".dds") || path.endsWith(".das") || path.endsWith(".dods"))
      return ServiceType.OPENDAP;

    if (path.endsWith(".dmr") || path.endsWith(".dap") || path.endsWith(".dsr"))
      return ServiceType.DAP4;

    if (path.endsWith(".xml") || path.endsWith(".ncml"))
      return ServiceType.NCML;
    return null;
  }

  /*
 * Attempt to map a leading url protocol url to a service type (see thredds.catalog.ServiceType).
 * Possible service types should include at least the following.
 * <ol>
 * <li> OPENDAP (DAP2 protocol)
 * <li> DAP4 (DAP4 protocol)
 * <li> CdmRemote (remote ncstream)
 * </ol>
 *
 * @param protocol The leading protocol
 * @return ServiceType indicating how to handle the url, or null.
 */
  @Urlencoded
  static private ServiceType decodeLeadProtocol(String protocol) throws IOException {
    if (protocol.equals("dods"))
      return ServiceType.OPENDAP;

    else if (protocol.equals("dap4"))
      return ServiceType.DAP4;

    else if (protocol.equals("httpserver") || protocol.equals("nodods"))
      return ServiceType.HTTPServer;

    else if (protocol.equals(CdmRemote.PROTOCOL))
      return ServiceType.CdmRemote;

    else if (protocol.equals(DataFactory.PROTOCOL)) //thredds
      return ServiceType.THREDDS;

    return null;
  }

  //////////////////////////////////////////////////////////////////

  /**
   * If the URL alone is not sufficient to disambiguate the location,
   * then this method will attempt to do a specific kind of request on
   * the server, typically a HEAD call using the URL.
   * It finds the header "Content-Description"
   * and uses it value (e.g. "ncstream" or "dods", etc)
   * in order to disambiguate.
   *
   * @param location the url to disambiguate
   * @return ServiceType indicating how to handle the url
   */
  @Urlencoded
  static private ServiceType disambiguateHttp(String location) throws IOException {
    boolean checkDap2 = false, checkDap4 = false, checkCdmr = false;

    // some TDS specific tests
    if (location.contains("cdmremote")) {
      ServiceType result = checkIfCdmr(location);
      if (result != null) return result;
      checkCdmr = true;
    }
    if (location.contains("dodsC")) {
      ServiceType result = checkIfDods(location);
      if (result != null) return result;
      checkDap2 = true;
    }

    if (location.contains("dap4")) {
      ServiceType result = checkIfDap4(location);
      if (result != null) return result;
      checkDap4 = true;
    }

    if (!checkDap2) {
      ServiceType result = checkIfDods(location);
      if (result != null)
        return result;
    }

    if (!checkDap4) {
      ServiceType result = checkIfDap4(location);
      if (result != null)
        return result;
    }

    if (!checkCdmr) {
      ServiceType result = checkIfCdmr(location);
      if (result != null)
        return result;
    }
    return null;
  }

  // cdmremote
  static private ServiceType checkIfCdmr(String location) throws IOException {

    try (HTTPMethod method = HTTPFactory.Head(location + "?req=header")) {
      int statusCode = method.execute();
      if (statusCode >= 300) {
        if (statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == HttpStatus.SC_FORBIDDEN)
          throw new IOException("Unauthorized to open dataset " + location);
        else
          throw new IOException(location + " is not a valid URL, return status=" + statusCode);
      }

      Header h = method.getResponseHeader("Content-Description");
      if ((h != null) && (h.getValue() != null)) {
        String v = h.getValue();
        if (v.equalsIgnoreCase("ncstream"))
          return ServiceType.CdmRemote;
      }
    }
    return null;
  }

  // not sure what other opendap servers do, so fall back on check for dds
  static private ServiceType checkIfDods(String location) throws IOException {
    int len = location.length();
    // Strip off any trailing .dds, .das, or .dods
    if (location.endsWith(".dds"))
      location = location.substring(0, len - ".dds".length());
    if (location.endsWith(".das"))
      location = location.substring(0, len - ".das".length());
    if (location.endsWith(".dods"))
      location = location.substring(0, len - ".dods".length());

    // Opendap assumes that the caller has properly escaped the url
    try (
            // For some reason, the head method is not using credentials
            // method = session.newMethodHead(location + ".dds");
            HTTPMethod method = HTTPFactory.Get(location + ".dds")) {

      int status = method.execute();
      if (status == 200) {
        Header h = method.getResponseHeader("Content-Description");
        if ((h != null) && (h.getValue() != null)) {
          String v = h.getValue();
          if (v.equalsIgnoreCase("dods-dds") || v.equalsIgnoreCase("dods_dds"))
            return ServiceType.OPENDAP;
          else
            throw new IOException("OPeNDAP Server Error= " + method.getResponseAsString());
        }
      }
      if (status == HttpStatus.SC_UNAUTHORIZED || status == HttpStatus.SC_FORBIDDEN)
        throw new IOException("Unauthorized to open dataset " + location);

      // not dods
      return null;
    }
  }

  // check for dmr
  static private ServiceType checkIfDap4(String location) throws IOException {
    // Strip off any trailing DAP4 prefix
    if (location.endsWith(".dap"))
      location = location.substring(0, location.length() - ".dap".length());
    else if (location.endsWith(".dmr"))
      location = location.substring(0, location.length() - ".dmr".length());
    else if (location.endsWith(".dmr.xml"))
      location = location.substring(0, location.length() - ".dmr.xml".length());
    else if (location.endsWith(".dsr"))
      location = location.substring(0, location.length() - ".dsr".length());
    try (HTTPMethod method = HTTPFactory.Get(location + ".dmr.xml")) {
      int status = method.execute();
      if (status == 200) {
        Header h = method.getResponseHeader("Content-Type");
        if ((h != null) && (h.getValue() != null)) {
          String v = h.getValue();
          if (v.startsWith("application/vnd.opendap.org"))
            return ServiceType.DAP4;
        }
      }
      if (status == HttpStatus.SC_UNAUTHORIZED || status == HttpStatus.SC_FORBIDDEN)
        throw new IOException("Unauthorized to open dataset " + location);

      // not dods
      return null;
    }
  }

  // The first 128 bytes should contain enough info to tell if this looks like an actual ncml file or not.
  // For example, here is an example 128 byte response:
  // <?xml version="1.0" encoding="UTF-8"?>\n<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" location="dods://ma
  private static int NUM_BYTES_TO_DETERMINE_NCML = 128;

  static private boolean checkIfRemoteNcml(String location) throws IOException {
    if (decodePathExtension(location)==ServiceType.NCML) {
      // just because location ends with ncml does not mean it's ncml
      // if the ncml file is being served up via http by a remote server,
      // we should be able to read the first bit of it and see if it even
      // looks like an ncml file.
      try (HTTPMethod method = HTTPFactory.Get(location)) {
        method.setRange(0, NUM_BYTES_TO_DETERMINE_NCML);
        method.setRequestHeader("accept-encoding", "identity");
        int statusCode = method.execute();
        if (statusCode >= 300) {
          if (statusCode == 401) {
            throw new IOException("Unauthorized to open dataset " + location);
          } else if (statusCode == 406) {
            String msg = location + " - this server does not support returning content without any encoding.";
            msg = msg + " Please download the file locally. Return status=" + statusCode;
            throw new IOException(msg);
          } else {
            throw new IOException(location + " is not a valid URL, return status=" + statusCode);
          }
        }

        return checkIfNcml(method.getResponseAsString());
      }
    }

    return false;
  }

  static private boolean checkIfNcml(File file) throws IOException {
    if (!file.exists()) {
      return false;
    }

    try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file), NUM_BYTES_TO_DETERMINE_NCML)) {
      byte[] bytes = new byte[NUM_BYTES_TO_DETERMINE_NCML];
      int bytesRead = in.read(bytes);

      if (bytesRead <= 0) {
        return false;
      } else {
        return checkIfNcml(new String(bytes, 0, bytesRead));
      }
    }
  }

  static private boolean checkIfNcml(String string) {
    // Look for the ncml element as well as a reference to the ncml namespace URI.
    return string.contains("<netcdf ") && string.contains("unidata.ucar.edu/namespaces/netcdf/ncml");
  }

  /////////////////////////////////////////////////////////////////////
  public final ServiceType serviceType;
  public final String trueurl;

  public DatasetUrl(ServiceType serviceType, String trueurl) {
    this.serviceType = serviceType;
    this.trueurl = trueurl;
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) {
      return true;
    }
    if (o == null || getClass() != o.getClass()) {
      return false;
    }
    DatasetUrl that = (DatasetUrl) o;
    return serviceType == that.serviceType && Objects.equals(trueurl, that.trueurl);
  }

  @Override
  public int hashCode() {
    return Objects.hash(serviceType, trueurl);
  }
}


/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */
package ucar.nc2.stream;

import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;

import java.io.*;
import java.nio.ByteOrder;

/**
 * Write a NetcdfFile to an OutputStream using ncstream protocol
 *
 * @author caron
 * @since Feb 7, 2009
 */
public class NcStreamWriter {
  static private long maxChunk = 1000 * 1000; // 1 MByte
  static private final int sizeToCache = 100; // when to store a variable's data in the header, ie "immediate" mode
  static private final int currentVersion = 1;

  private NetcdfFile ncfile;
  private NcStreamProto.Header header;
  private boolean show = false;

  public NcStreamWriter(NetcdfFile ncfile, String location) throws IOException {
    this.ncfile = ncfile;
    NcStreamProto.Group.Builder rootBuilder = NcStream.encodeGroup(ncfile.getRootGroup(), sizeToCache);

    NcStreamProto.Header.Builder headerBuilder = NcStreamProto.Header.newBuilder();
    headerBuilder.setLocation(location == null ? ncfile.getLocation() : location);
    if (ncfile.getTitle() != null) headerBuilder.setTitle(ncfile.getTitle());
    if (ncfile.getId() != null) headerBuilder.setId(ncfile.getId());
    headerBuilder.setRoot(rootBuilder);
    headerBuilder.setVersion(currentVersion);

    header = headerBuilder.build();
  }

  public long sendStart(OutputStream out) throws IOException {
    return writeBytes(out, NcStream.MAGIC_START);
  }

  public long sendEnd(OutputStream out) throws IOException {
    return writeBytes(out, NcStream.MAGIC_END);
  }

  public long sendHeader(OutputStream out) throws IOException {
    long size = 0;

    //// header message
    size += writeBytes(out, NcStream.MAGIC_HEADER);
    byte[] b = header.toByteArray();
    size += NcStream.writeVInt(out, b.length); // len
    if (show) System.out.println("Write Header len=" + b.length);

    // payload
    size += writeBytes(out, b);
    if (show) System.out.println(" header size=" + size);

    return size;
  }

  public long sendData(Variable v, Section section, OutputStream out, NcStreamCompression compress) throws IOException, InvalidRangeException {
    if (show) System.out.printf(" %s section=%s%n", v.getFullName(), section);

    // length of data uncompressed
    long uncompressedLength = section.computeSize();
    if ((v.getDataType() != DataType.STRING) && (v.getDataType() != DataType.OPAQUE) && !v.isVariableLength())
      uncompressedLength *= v.getElementSize(); // nelems for vdata, else nbytes

    ByteOrder bo = ByteOrder.nativeOrder(); // reader makes right
    long size = 0;
    size += writeBytes(out, NcStream.MAGIC_DATA); // magic
    NcStreamProto.Data dataProto = NcStream.encodeDataProto(v, section, compress.type, bo, (int) uncompressedLength);
    byte[] datab = dataProto.toByteArray();
    size += NcStream.writeVInt(out, datab.length); // dataProto len
    size += writeBytes(out, datab); // dataProto

    // version < 3
    if (v.getDataType() == DataType.SEQUENCE) {
      assert (v instanceof Structure);
      int count = 0;
      Structure seq = (Structure) v; // superclass for Sequence, SequenceDS
      //coverity[FB.BC_UNCONFIRMED_CAST]
      try (StructureDataIterator iter = seq.getStructureIterator(-1)) {
        while (iter.hasNext()) {
          size += writeBytes(out, NcStream.MAGIC_VDATA); // magic
          StructureData sdata = iter.next();
          ArrayStructure as = new ArrayStructureW(sdata);
          size += NcStream.encodeArrayStructure(as, bo, out);
          count++;
        }
      }
      size += writeBytes(out, NcStream.MAGIC_VEND);
      if (show) System.out.printf(" NcStreamWriter sent %d seqData bytes = %d%n", count, size);
      return size;
    }

    // version < 3
    if (v.getDataType() == DataType.STRUCTURE) {
      ArrayStructure abb = (ArrayStructure) v.read();   // read all - LOOK break this up into chunks if needed
      //coverity[FB.BC_UNCONFIRMED_CAST]
      size += NcStream.encodeArrayStructure(abb, bo, out);
      if (show) System.out.printf(" NcStreamWriter sent ArrayStructure bytes = %d%n", size);
      return size;
    }

    // Writing the size of the block is handled for us.
    out = compress.setupStream(out, (int)uncompressedLength);
    size += v.readToStream(section, out);
    out.flush();
    return size;
  }

  // LOOK compression not used
  public long sendData2(Variable v, Section section, OutputStream out, NcStreamCompression compress) throws IOException, InvalidRangeException {
    if (show) System.out.printf(" %s section=%s%n", v.getFullName(), section);

    boolean isVlen = v.isVariableLength(); //  && v.getRank() > 1;
    if (isVlen)
      v.read(section);
    NcStreamDataCol encoder = new NcStreamDataCol();
    NcStreamProto.DataCol dataProto = encoder.encodeData2(v.getFullName(), isVlen, section, v.read(section));

    // LOOK trap error, write error message ??

    // dataProto.writeDelimitedTo(out);
    long size = 0;
    size += writeBytes(out, NcStream.MAGIC_DATA2); // data version 3

    byte[] datab = dataProto.toByteArray();
    size += NcStream.writeVInt(out, datab.length); // dataProto len
    size += writeBytes(out, datab); // dataProto
    return size;
  }

  private int writeBytes(OutputStream out, byte[] b) throws IOException {
    out.write(b);
    return b.length;
  }

  public long streamAll(OutputStream out) throws IOException, InvalidRangeException {
    long size = writeBytes(out, NcStream.MAGIC_START);
    size += sendHeader(out);
    if (show) System.out.printf(" data starts at= %d%n", size);

    for (Variable v : ncfile.getVariables()) {
      NcStreamCompression compress;
      Attribute compressAtt = v.findAttribute(CDM.COMPRESS);
      if (compressAtt != null && compressAtt.isString()) {
        String compType = compressAtt.getStringValue();
        if (compType.equalsIgnoreCase(CDM.COMPRESS_DEFLATE)) {
          compress = NcStreamCompression.deflate();
        } else {
          if (show) System.out.printf(" Unknown compression type %s. Defaulting to none.%n", compType);
          compress = NcStreamCompression.none();
        }
      } else {
        compress = NcStreamCompression.none();
      }

      long vsize = v.getSize() * v.getElementSize();
      //if (vsize < sizeToCache) continue; // in the header;
      if (show) System.out.printf(" var %s len=%d starts at= %d%n", v.getFullName(), vsize, size);

      if (vsize > maxChunk) {
        size += copyChunks(out, v, maxChunk, compress);
      } else {
        size += sendData(v, v.getShapeAsSection(), out, compress);
      }
    }

    size += writeBytes(out, NcStream.MAGIC_END);
    if (show) System.out.printf("total size= %d%n", size);
    return size;
  }

  private long copyChunks(OutputStream out, Variable oldVar, long maxChunkSize, NcStreamCompression compress) throws IOException {
    long maxChunkElems = maxChunkSize / oldVar.getElementSize();
    FileWriter2.ChunkingIndex index = new FileWriter2.ChunkingIndex(oldVar.getShape());
    long size = 0;
    while (index.currentElement() < index.getSize()) {
      try {
        int[] chunkOrigin = index.getCurrentCounter();
        int[] chunkShape = index.computeChunkShape(maxChunkElems);
        size += sendData(oldVar, new Section(chunkOrigin, chunkShape), out, compress);
        index.setCurrentCounter(index.currentElement() + (int) Index.computeSize(chunkShape));

      } catch (InvalidRangeException e) {
        e.printStackTrace();
        throw new IOException(e.getMessage());
      }
    }
    return size;
  }

}