def test_use_list_options(foo_nc):
    nco = Nco(debug=True)
    options = []
    options.extend(['-a', 'units,time,o,c,days since 1999-01-01'])
    options.extend(['-a', 'long_name,time,o,c,time'])
    options.extend(['-a', 'calendar,time,o,c,noleap'])
    nco.ncrcat(input=foo_nc, output='out.nc', options=options)

def test_returnArray(foo_nc):
    nco = Nco(cdfMod='netcdf4')
    random1 = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O']).variables['random'][:]
    assert type(random1) == np.ndarray
    random2 = nco.ncea(input=foo_nc, output="tmp.nc",returnArray='random' ,options=['-O'])
    assert type(random2) == np.ndarray
    np.testing.assert_equal(random1, random2)

def normalize_time(netcdf_file):
    epoch_units       = 'seconds since 1970-01-01T00:00:00Z'
    millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z'

    with nc4.Dataset(netcdf_file, 'a') as nc:
        # Signell said this works, any problems and we can all blame him!
        time_data = nc4.num2date(
            (
                np.int64(nc.variables['time'][:]) - 2400001
            ) * 3600 * 24 * 1000 + nc.variables['time2'][:].__array__(),
            units=millisecond_units
        )
        nc.renameVariable("time", "old_time")
        nc.sync()

        time = nc.createVariable('time', 'f8', ('time'))
        time.units          = epoch_units
        time.standard_name  = "time"
        time.long_name      = "time of measurement"
        time.calendar       = "gregorian"
        time.axis           = "T"
        time[:] = nc4.date2num(time_data, units=epoch_units).round()

    o = Nco()
    o.ncks(
        input=netcdf_file,
        output=netcdf_file,
        options=[
            '-O',
            '-h',
            '-x',
            '-v', 'time2,old_time'
        ]
    )

def test_use_list_options(foo_nc):
    nco = Nco(debug=True)
    options = []
    options.extend(["-a", 'units,time,o,c,"days since 1999-01-01"'])
    options.extend(["-a", "long_name,time,o,c,time"])
    options.extend(["-a", "calendar,time,o,c,noleap"])
    nco.ncatted(input=foo_nc, output="out.nc", options=options)

def normalize_ctd_depths(netcdf_file):
    with CFDataset(netcdf_file, 'a') as nc:
        depths = nc.variables['depth'][:][0]
        z_atts = nc.vatts('depth')

    # Remove the old depth variable so we can add a new one with no dimensions
    o = Nco()
    o.ncks(
        input=netcdf_file,
        output=netcdf_file,
        options=[
            '-O',
            '-h',
            '-x',
            '-v', 'depth'
        ]
    )

    # Add the new Z variable
    with nc4.Dataset(netcdf_file, 'a') as nc:
        z = nc.createVariable('depth', 'f4')
        z[:] = depths
        z_atts.update({
            'standard_name': 'depth',
            'axis': 'Z',
            'positive': 'down'
        })
        z.setncatts(z_atts)

def test_return_array(foo_nc):
    nco = Nco(cdf_module="netcdf4")
    random1 = nco.ncea(
        input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"]
    ).variables["random"][:]
    assert isinstance(random1, np.ndarray)
    random2 = nco.ncea(
        input=foo_nc, output="tmp.nc", returnArray="random", options=["-O"]
    )
    assert isinstance(random2, np.ndarray)
    np.testing.assert_equal(random1, random2)

def test_command_line_options(foo_nc):
    """
    3.4 Command Line Options

    ncks -D 3 in.nc        # Short option
    ncks --dbg_lvl=3 in.nc # Long option, preferred form
    ncks --dbg_lvl 3 in.nc # Long option, alternate form
    """
    nco = Nco(debug=True)
    nco.ncks(input=foo_nc, options=["-O -D 3"])
    nco.ncks(input=foo_nc, options=["-O --dbg_lvl=3"])
    nco.ncks(input=foo_nc, options=["-O --dbg_lvl 3"])

def normalize_netcdf4(netcdf_file):
    o = Nco()
    o.ncks(
        input=netcdf_file,
        output=netcdf_file,
        options=[
            '-O',
            '-h',
            '-4',
            '-L3'
        ]
    )

def test_ncks_hdf2nc(hdf_file):
    """
    1.6 netCDF2/3/4 and HDF4/5 Support
    Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively,
    it is now easy to convert HDF4 files to netCDF files directly, e.g.,

    ncks        fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+)
    ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9)
    """
    nco = Nco(debug=True)
    nco.ncks(input=hdf_file, output='foo.nc')
    nco.ncks(input=hdf_file, output='foo.nc', hdf4=True)

def test_returnCdf(foo_nc):
    nco = Nco(cdfMod='scipy')
    testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True,options=['-O'])
    assert type(testCdf) == scipy.io.netcdf.netcdf_file
    expected_vars = ['time', 'random']
    for var in expected_vars:
        assert var in list(testCdf.variables.keys())

    nco = Nco(cdfMod='netcdf4')
    testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O'])
    assert type(testCdf) == netCDF4.Dataset
    for var in expected_vars:
        assert var in list(testCdf.variables.keys())

def test_return_cdf(foo_nc):
    nco = Nco(cdf_module="scipy")
    test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"])
    assert type(test_cdf) == scipy.io.netcdf.netcdf_file
    expected_vars = ["time", "random"]
    for var in expected_vars:
        assert var in list(test_cdf.variables.keys())

    nco = Nco(cdf_module="netcdf4")
    test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"])
    assert type(test_cdf) == netCDF4.Dataset
    for var in expected_vars:
        assert var in list(test_cdf.variables.keys())

def test_ncks_hdf2nc(hdf_file):
    """
    1.6 netCDF2/3/4 and HDF4/5 Support
    Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively,
    it is now easy to convert HDF4 files to netCDF files directly, e.g.,

    ncks        fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+)
    ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9)
    """
    if hdf_file is None:
        pytest.skip("Skipped because h5py is not installed")
    nco = Nco(debug=True)
    nco.ncks(input=hdf_file, output="foo.nc")
    nco.ncks(input=hdf_file, output="foo.nc", hdf4=True)

def test_ncks_append_variables(foo_nc, bar_nc):
    """
    2.4 Appending Variables
    The simplest way to create the union of two files is

    ncks -A fl_1.nc fl_2.nc
    """
    nco = Nco(debug=True)
    nco.ncks(input=foo_nc, output=bar_nc, options=['-A'])
    nco.ncks(input=foo_nc, output=bar_nc, append=True)
    nco.ncks(input=foo_nc, output=bar_nc, apn=True)
    nco.ncks(input=foo_nc, output=bar_nc)

def xgeo_multifile_load(nc_wildcard, nc_dir=None):
    """
    Read data from multiple (e.g. hourly xgeo data) netcdf files
    :param nc_wildcard: common section of the filenams, e.g. mf_files*.nc
    :return:
    """
    if dir:
        nc_path = os.path.join(nc_dir, nc_wildcard)
    else:
        nc_path = nc_wildcard

    nco = Nco()
    nc_temp = nco.ncrcat(input=nc_path)
    nc = netCDF4.Dataset(nc_temp)

    #add function to restrict dates and times
    return nc

def test_specifying_input_files(testfiles8589):
    """
    3.5 Specifying Input Files

    ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra 8[56789].nc 8589.nc
    ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra -n 5,2,1 85.nc 8589.nc
    """
    inputs = ['85.nc', '86.nc', '87.nc', '88.nc', '89.nc']
    nco = Nco(debug=True)
    nco.ncra(input=inputs, ouptut='8589.nc')
    nco.ncra(input='8[56789].nc', ouptut='8589.nc')
    srcdir = os.path.split(inputs[0])
    nco.ncra(input=inputs, ouptut='8589.nc', path=srcdir)
    nco.ncra(input=inputs, ouptut='8589.nc', nintap='5,2,1')

def normalize_locations(netcdf_file):
    with CFDataset(netcdf_file) as nc:
        y = nc.variables.get("lat")
        y_data = y[:]
        y_atts = nc.vatts('lat')

        x = nc.variables.get("lon")
        x_data = x[:]
        x_atts = nc.vatts('lon')

    # Remove the old x/y variable so we can add new ones with no dimensions
    o = Nco()
    o.ncks(
        input=netcdf_file,
        output=netcdf_file,
        options=[
            '-O',
            '-h',
            '-x',
            '-v', 'lat,lon'
        ]
    )

    # Add the new X/Y variables
    with nc4.Dataset(netcdf_file, 'a') as nc:
        lat = nc.createVariable('lat', y_data.dtype)
        lat[:] = y_data
        y_atts.update({
            'standard_name': 'latitude',
            'axis': 'Y'
        })
        lat.setncatts(y_atts)

        lon = nc.createVariable('lon', x_data.dtype)
        lon[:] = x_data
        x_atts.update({
            'standard_name': 'longitude',
            'axis': 'X'
        })
        lon.setncatts(x_atts)

def test_specifying_input_files(testfiles8589):
    """
    3.5 Specifying Input Files

    ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra 8[56789].nc 8589.nc
    ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra -n 5,2,1 85.nc 8589.nc
    """
    nco = Nco(debug=True)
    nco.ncra(input=testfiles8589, output='8589.nc')
    nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1')

    srcdir = os.path.dirname(testfiles8589[0])
    basenames = [ os.path.basename(x) for x in testfiles8589 ]
    nco.ncra(input=basenames, output='8589.nc', path=srcdir)

    regpath = os.path.join(srcdir, '8[56789].nc')
    nco.ncra(input=regpath, output='8589.nc')

def test_specifying_input_files(testfiles8589):
    """
    3.5 Specifying Input Files

    ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra 8[56789].nc 8589.nc
    ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
    ncra -n 5,2,1 85.nc 8589.nc
    """
    nco = Nco(debug=True)
    nco.ncra(input=testfiles8589, output="8589.nc")
    nco.ncra(input=testfiles8589, output="8589.nc", nintap="5,2,1")

    srcdir = os.path.dirname(testfiles8589[0])
    basenames = [os.path.basename(x) for x in testfiles8589]
    nco.ncra(input=basenames, output="8589.nc", path=srcdir)

    # unable to use brackets, perhaps because we're no longer using shell=True when calling subprocess()?
    regpath = os.path.join(srcdir, "8[56789].nc")
    nco.ncra(input=regpath, output="8589.nc", use_shell=True)

    def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
        """ Combine many files into a single file on disk.  Defaults to using the 'time' dimension. """
        nco = None
        try:
            nco = Nco()
        except BaseException:
            raise ImportError("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")

        if len(members) > 0 and hasattr(members[0], 'path'):
            # A member DotDoct was passed in, we only need the paths
            members = [ m.path for m in members ]

        options  = ['-4']  # NetCDF4
        options += ['-L', '3']  # Level 3 compression
        options += ['-h']  # Don't append to the history global attribute
        if dimension is not None:
            if start_index is None:
                start_index = 0
            if stop_index is None:
                stop_index = ''
            if stride is None:
                stride = 1
            options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
        nco.ncrcat(input=members, output=output_file, options=options)

def dailyAve():
    from nco import Nco
    import datetime
    nco = Nco()
    for d in range(365):
        dp =  datetime.date(startY,1,1)+datetime.timedelta(d)
        print "Averaging TRMM 3B42 for day "+dp.strftime('%j')+"..."
        ifile = ' '.join("3B42_daily."+str(year)+"."+dp.strftime('%m')+"."+dp.strftime('%d')+".7.nc" for year in range(startY,endY))
        ofile = "3B42_aver."+dp.strftime('%j')+".nc"           
        if not os.path.isfile(ofile):
            nco.ncra(input=ifile, output=ofile)
    nco.ncrcat(input="3B42_aver.*.nc", output="3B42_cat.nc", options="-d time,1,365")
    nco.ncwa(input="3B42_cat.nc", output="3B42_MAP.nc", options='-N -a time')

    return None