def drifter_meta():
    imei = {}
    wmo = {}
    deployment = {}

    with Dataset(app.config['DRIFTER_AGG_URL'], 'r') as ds:
        for idx, b in enumerate(ds.variables['buoy'][:]):
            bid = str(chartostring(b))[:-3]

            for data, key in [
                [imei, 'imei'],
                [wmo, 'wmo'],
                [deployment, 'deployment']
            ]:
                d = str(chartostring(ds.variables[key][idx][0]))
                if data.get(d) is None:
                    data[d] = [bid]
                else:
                    data[d].append(bid)

    return {
        'imei': imei,
        'wmo': wmo,
        'deployment': deployment,
    }

    def __str__(self):
        return """{0}

Creation Status: {1}
Source Seed: {2}\tSource NN: {3}\tSource NB: {4}\t Source Lp: {5}
Box Matrix: {6}
Box String: {7}
Radis: {8}
Number of Nodes: {9}\tNumber of Surface Nodes: {10}\tNumber of Bonds {11}
Source Z: {12}\tSource Delta Z: {13}
Z: {14}\tDelta Z: {15}
""".format(
            self.data,
            chartostring(self.get("creation_status", 0)),
            self.get("seed_src", 0),
            self.get("NN_src", 0),
            self.get("NB_src", 0),
            self.get("Lp_src", 0),
            self.get("box_mat", 0),
            chartostring(self.get("box_str", 0)),
            self.get("radius", 0),
            self.get("NN", 0),
            self.get("NSN", 0),
            self.get("NB", 0),
            self.get("Z_src", 0),
            self.get("DeltaZ_src", 0),
            self.get("Z", 0),
            self.get("DeltaZ", 0),
        )

def observation_meta():
    with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds:
        ship = chartostring(ds['ship'][:])
        trip = chartostring(ds['trip'][:])

        data = {
            'ship': sorted(np.unique(ship).tolist()),
            'trip': sorted(np.unique(trip).tolist()),
        }

        return data

 def runTest(self):
     """testing functions for converting arrays of chars to fixed-len strings"""
     nc = Dataset(FILE_NAME)
     assert nc.dimensions['n1'].isunlimited() == True
     v = nc.variables['strings']
     v2 = nc.variables['strings2']
     v3 = nc.variables['strings3']
     assert v.dtype.str[1:] in ['S1','U1']
     assert v.shape == (nrecs,n2,nchar)
     for nrec in range(nrecs):
         data2 = chartostring(v[nrec],encoding='ascii')
         assert_array_equal(data2,datau[nrec])
     data2 = v2[:]
     data2[0] = v2[0]
     data2[0,1] = v2[0,1]
     assert_array_equal(data2,datau)
     data3 = v3[:]
     assert_array_equal(data3,datau)
     # these slices should return a char array, not a string array
     data4 = v2[:,:,0]
     assert(data4.dtype.itemsize == 1)
     assert_array_equal(data4, datac[:,:,0])
     data5 = v2[0,0:nchar,0]
     assert(data5.dtype.itemsize == 1)
     assert_array_equal(data5, datac[0,0:nchar,0])
     # test turning auto-conversion off.
     v2.set_auto_chartostring(False)
     data6 = v2[:]
     assert(data6.dtype.itemsize == 1)
     assert_array_equal(data6, datac)
     nc.set_auto_chartostring(False)
     data7 = v3[:]
     assert(data7.dtype.itemsize == 1)
     assert_array_equal(data7, datac)
     nc.close()

def list_class4(d):
    dataset_url = app.config["CLASS4_URL"] % d

    with Dataset(dataset_url, 'r') as ds:
        lat = ds['latitude'][:]
        lon = ds['longitude'][:]
        ids = map(str.strip, chartostring(ds['id'][:]))
        rmse = []

        for i in range(0, lat.shape[0]):
            best = ds['best_estimate'][i, 0, :]
            obsv = ds['observation'][i, 0, :]
            rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean()))

    rmse = np.ma.hstack(rmse)
    maxval = rmse.mean() + 2 * rmse.std()
    rmse_norm = rmse / maxval

    loc = zip(lat, lon)

    points = []
    for idx, ll in enumerate(loc):
        if np.ma.is_masked(rmse[idx]):
            continue
        points.append({
            'name': "%s" % ids[idx],
            'loc': "%f,%f" % ll,
            'id': "%s/%d" % (d, idx),
            'rmse': float(rmse[idx]),
            'rmse_norm': float(rmse_norm[idx]),
        })
        points = sorted(points, key=lambda k: k['id'])

    return points

def class4(class4_id, projection, resolution, extent):
    dataset_url = app.config["CLASS4_URL"] % class4_id

    proj = pyproj.Proj(init=projection)
    view = _get_view(extent)

    rmse = []
    lat = []
    lon = []
    point_id = []
    identifiers = []
    with Dataset(dataset_url, 'r') as ds:
        lat_in = ds['latitude'][:]
        lon_in = ds['longitude'][:]
        ids = map(str.strip, chartostring(ds['id'][:]))

        for i in range(0, lat_in.shape[0]):
            x, y = proj(lon_in[i], lat_in[i])
            p = Point(y, x)
            if view.envelope.intersects(p):
                lat.append(float(lat_in[i]))
                lon.append(float(lon_in[i]))
                identifiers.append(ids[i])
                best = ds['best_estimate'][i, 0, :]
                obsv = ds['observation'][i, 0, :]
                point_id.append(i)
                rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean()))

    rmse = np.ma.hstack(rmse)
    rmse_norm = np.clip(rmse / 1.5, 0, 1)

    loc = zip(lon, lat)

    points = []

    for idx, ll in enumerate(loc):
        if np.ma.is_masked(rmse[idx]):
            continue
        points.append({
            'type': "Feature",
            'geometry': {
                'type': "Point",
                'coordinates': ll,
            },
            'properties': {
                'name': "%s" % identifiers[idx],
                'id': "%s/%d" % (class4_id, point_id[idx]),
                'error': float(rmse[idx]),
                'error_norm': float(rmse_norm[idx]),
                'type': 'class4',
                'resolution': 0,
            },
        })

    result = {
        'type': "FeatureCollection",
        'features': points,
    }

    return result

def get_data_from_file(path, field_name = 'water_discharge'):
    """
    returns streamflow, times, x_indices, y_indices
    """
    date_format = '%Y_%m_%d_%H_%M'
    print path
    ds = nc.Dataset(path)

    #dims: time, cell_index
    discharge = ds.variables[field_name][:]

    times = ds.variables['time'][:]
    times = chartostring(times)


    if ds.variables.has_key('x_index'):
        x_indices = ds.variables['x_index'][:]
        y_indices = ds.variables['y_index'][:]
    else:
        x_indices = ds.variables['x-index'][:]
        y_indices = ds.variables['y-index'][:]


    date_times = map(lambda x: datetime.strptime( x , date_format ), times)



    return discharge, date_times, x_indices, y_indices

def main():
    global ncfile, prefix

    parser = argparse.ArgumentParser(description="A commandline tool to convert WRF timeseries files to netCDF4")
    parser.add_argument('netcdf', metavar="netCDF4 file", type=str, nargs=1, help="NetCDF file base name. Create it first with ts_make_ncs.sh")
    parser.add_argument('-d', '--domain', metavar="domain", type=int, nargs=1, help="WRF domain number", default=0)
    args = parser.parse_args()

    for varname in ['TS']: # ,'UU','VV','TH','QV','PH']:
        logging.info( "{}".format(varname) )
        ncfile = cdf.Dataset( args.netcdf[0] + "." + varname + ".nc", "r+" )
        do_tslist()
        prefix  = ncfile.variables['prefix']

        for stationi in range(nstations):

            filename = "{}.d{:02d}.".format( cdf.chartostring( prefix[stationi] ), args.domain[0] )
            if not os.path.isfile( filename + "TS" ):
                continue 

            if varname in ['TS',]:
                if not ntimes:
                    simplecount(filename + "TS", args.domain[0])

                do_tsfile ( filename + "TS", stationi )
            else:
                do_profile( filename + varname, stationi, varname )

        if varname in ['TS',]:
            flush_tsfile()
        else:
            ncfile.variables[varname][:,:,:]     = profile   [:,:,:]

        ncfile.close()

def maketransectdf(transect, dt_from=None, dt_to=None):
    """Read some transect data"""
    transectidx, filter, vardict, ds = prepare_vardict(transect, 'transect', False, dt_from, dt_to)

    if transectidx is None:
        alongshore = np.zeros(0)
        areaname = np.zeros(0)
        mean_high_water = np.zeros(0)
        mean_low_water = np.zeros(0)
    else:
        alongshore = ds.variables['alongshore'][transectidx]
        areaname = str(netCDF4.chartostring(ds.variables['areaname'][transectidx])).strip()
        mean_high_water = ds.variables['mean_high_water'][transectidx]
        mean_low_water = ds.variables['mean_low_water'][transectidx]

    ds.close()
    transectdf = pandas.DataFrame(
        index=[transect],
        data=dict(
            transect=transect,
            areaname=areaname,
            mean_high_water=mean_high_water,
            mean_low_water=mean_low_water
        )
    )
    return transectdf

def do_tsfile(filename, stationi):

    if not os.path.isfile( filename):
        logging.info( "Skipping station %s : TS", cdf.chartostring( prefix[stationi] ) )
        return

    logging.debug( "{} starting".format( filename ) )

    logging.debug( "parsing ts file" )


    # Parse TS file, see the README.tslist in the WRF run directory for details

    fileTS = open( filename, 'r' )

    # Header
    # NZCM McMurdo               2  7 mcm   (-77.850, 166.710) ( 153, 207) (-77.768, 166.500)   81.8 meters
    # 
    # Those are name of the station, grid ID, time-series ID, station lat/lon, grid indices (nearest grid point to
    # the station location), grid lat/lon, elevation.

    line = fileTS.next()
    fields = re.search(r"\( *(\d+) *, *(\d+) *\) *\( *(-?\d+\.\d+) *, *(-?\d+\.\d+) *\) * (-?\d+(\.\d+)?) *meters", line).groups()

    ncfile.variables['gj'  ][stationi]      = fields[0]
    ncfile.variables['gi'  ][stationi]      = fields[1]
    ncfile.variables['glat'][stationi]      = fields[2]
    ncfile.variables['glon'][stationi]      = fields[3]
    ncfile.variables['elevation'][stationi] = fields[4]

    # Body
    # 0   1           2       3   4  5  6  7  8  9     10   11   12   13  14  15        16     17      18
    # id, ts_hour, id_tsloc, ix, iy, t, q, u, v, psfc, glw, gsw, hfx, lh, tsk, tslb(1), rainc, rainnc, clw
    timei = -1 
    for line in fileTS:
        timei += 1
        fields = line.split()
        time  [ timei ] = fields[ 1]
        T2m   [ timei,stationi ] = fields[ 5]
        Q2m   [ timei,stationi ] = fields[ 6]
        U10m  [ timei,stationi ] = fields[ 7]
        V10m  [ timei,stationi ] = fields[ 8]
        psfc  [ timei,stationi ] = fields[ 9]
        glw   [ timei,stationi ] = fields[10]
        gsw   [ timei,stationi ] = fields[11]
        hfx   [ timei,stationi ] = fields[12]
        lh    [ timei,stationi ] = fields[13]
        tsk   [ timei,stationi ] = fields[14]
        tslb1 [ timei,stationi ] = fields[15]
        rainc [ timei,stationi ] = fields[16]
        rainnc[ timei,stationi ] = fields[17]
        clw   [ timei,stationi ] = fields[18]
        tc2m  [ timei,stationi ] = fields[19]
        tp2m  [ timei,stationi ] = fields[20]

    fileTS.close()


    logging.info( "{} done".format( filename ) )

def xtimeGetYear(xtime):
  """Get an array of years from an xtime array, ignoring any partial year information"""
  # First parse the xtime character array into a string 
  xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function
  years = np.zeros( (len(xtimestr),) )
  for i in range(len(xtimestr)):
      years[i] = ( int(xtimestr[i].split('-')[0]) ) # Get the year part and make it an integer
  return years

def read_json(open_file, var_name):
    qp = nc.chartostring((open_file.variables[var_name][:]))

    try:
        parsed_json = json.loads(qp[0])
        return parsed_json
    except:
        parsed_json = 'n/a'
        return parsed_json

def test_sweep_mode():
    assert 'standard_name' in radar.sweep_mode
    assert radar.sweep_mode['data'].shape == (1, 32)
    assert radar.sweep_mode['data'].dtype.char == 'S'
    str_array = netCDF4.chartostring(radar.sweep_mode['data'])
    try:
        assert np.all(str_array == ['azimuth_surveillance'])
    except AssertionError:
        assert np.all(str_array == [b'azimuth_surveillance'])

def insert_station(sesh, cf, i, name, x, y):
    station = Station(
        name=str(chartostring(name[i])),
        x=x[i],
        x_units=x.units,
        y=y[i],
        y_units=y.units,
    )
    sesh.add(station)
    return station

def check_valid_time_format(section, dataset, text, var_name):
    """ Check that a variable is a valid UTC time formatted string. """
    if var_name in dataset.variables:
        s = str(netCDF4.chartostring(dataset.variables[var_name][:]))
        try:
            time.strptime(s, '%Y-%m-%dT%H:%M:%SZ')
        except:
            tup = (text, var_name, s, 'yyyy-mm-ddThh:mm:ssZ')
            t = "%s '%s' has an invalid format: %s should be %s" % (tup)
            log_error(section, t)

 def runTest(self):
     """testing functions for converting arrays of chars to fixed-len strings"""
     nc = Dataset(FILE_NAME)
     assert nc.dimensions['n1'].isunlimited() == True
     v = nc.variables['strings']
     assert v.dtype.str[1:] == 'S1'
     assert v.shape == (nrecs,n2,nchar)
     for nrec in range(nrecs):
         data2 = chartostring(v[nrec])
         assert_array_equal(data2,data[nrec])
     nc.close()

def check_dataset_metadata_in_storage_unit(nco, dataset_dir):
    assert len(nco.variables['dataset']) == 1  # 1 time slice
    stored_metadata = nco.variables['dataset'][0]
    if not isinstance(stored_metadata, str):
        stored_metadata = netCDF4.chartostring(stored_metadata)
        stored_metadata = str(np.char.decode(stored_metadata))
    ds_filename = dataset_dir / 'agdc-metadata.yaml'

    stored = yaml.safe_load(stored_metadata)
    [(_, original)] = read_documents(ds_filename)
    assert len(stored['lineage']['source_datasets']) == 1
    assert next(iter(stored['lineage']['source_datasets'].values())) == original

def find_station(sesh, cf, i, name, x, y):
    return (
        sesh.query(Station)
        .filter_by(
            name=str(chartostring(name[i])),
            x=x[i],
            x_units=x.units,
            y=y[i],
            y_units=y.units,
        )
        .first()
    )

def check_dataset_metadata_in_storage_unit(nco, dataset_dir):
    assert len(nco.variables['dataset']) == 1  # 1 time slice
    stored_metadata = nco.variables['dataset'][0]
    if not isinstance(stored_metadata, str):
        stored_metadata = netCDF4.chartostring(stored_metadata)
        stored_metadata = str(np.char.decode(stored_metadata))
    ds_filename = dataset_dir / 'agdc-metadata.yaml'
    with ds_filename.open() as f:
        orig_metadata = f.read()
    stored = make_pgsqljson_match_yaml_load(yaml.safe_load(stored_metadata))
    original = make_pgsqljson_match_yaml_load(yaml.safe_load(orig_metadata))
    assert len(stored['lineage']['source_datasets']) == 1
    assert next(iter(stored['lineage']['source_datasets'].values())) == original

def xtime2numtime(xtime):
  # First parse the xtime character array into a string 
  xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function

  dt = []
  for stritem in xtimestr:
      itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-')  # Get an array of strings that are Y,M,D,h,m,s
      results = [int(i) for i in itemarray]
      if (results[0] < 1900):  # datetime has a bug where years less than 1900 are invalid on some systems
         results[0] += 1900
      dt.append( datetime.datetime(*results) ) # * notation passes in the array as arguments

  numtime = netCDF4.date2num(dt, units='seconds since '+str(dt[0]))   # use the netCDF4 module's function for converting a datetime to a time number
  return numtime