def read(input_fname, gold_fname, output_fname=None, with_confidence=False,
         max_sent_size=_max_sent_size):
    inp = read_system_input(input_fname, max_sent_size=max_sent_size)
    gold = read_gold_standard(gold_fname)
    if output_fname:
        out = read_system_output(output_fname, with_confidence)
        return merge_arrays((inp, gold, out), flatten=True)
    else:
        return merge_arrays((inp, gold), flatten=True)

    def test_solo_w_flatten(self):
        # Test merge_arrays on a single array w & w/o flattening
        w = self.data[0]
        test = merge_arrays(w, flatten=False)
        assert_equal(test, w)

        test = merge_arrays(w, flatten=True)
        control = np.array([(1, 2, 3.0), (4, 5, 6.0)], dtype=[("a", int), ("ba", float), ("bb", int)])
        assert_equal(test, control)

 def test_wmasked_arrays(self):
     # Test merge_arrays masked arrays
     (_, x, _, _) = self.data
     mx = ma.array([1, 2, 3], mask=[1, 0, 0])
     test = merge_arrays((x, mx), usemask=True)
     control = ma.array([(1, 1), (2, 2), (-1, 3)], mask=[(0, 1), (0, 0), (1, 0)], dtype=[("f0", int), ("f1", int)])
     assert_equal(test, control)
     test = merge_arrays((x, mx), usemask=True, asrecarray=True)
     assert_equal(test, control)
     assert_(isinstance(test, MaskedRecords))

    def test_flatten(self):
        # Test standard & flexible
        (_, x, _, z) = self.data
        test = merge_arrays((x, z), flatten=True)
        control = np.array([(1, "A", 1.0), (2, "B", 2.0)], dtype=[("f0", int), ("A", "|S3"), ("B", float)])
        assert_equal(test, control)

        test = merge_arrays((x, z), flatten=False)
        control = np.array(
            [(1, ("A", 1.0)), (2, ("B", 2.0))], dtype=[("f0", int), ("f1", [("A", "|S3"), ("B", float)])]
        )
        assert_equal(test, control)

    def test_flatten_wflexible(self):
        # Test flatten standard & nested
        (w, x, _, _) = self.data
        test = merge_arrays((x, w), flatten=True)
        control = np.array(
            [(1, 1, 2, 3.0), (2, 4, 5, 6.0)], dtype=[("f0", int), ("a", int), ("ba", float), ("bb", int)]
        )
        assert_equal(test, control)

        test = merge_arrays((x, w), flatten=False)
        controldtype = [("f0", int), ("f1", [("a", int), ("b", [("ba", float), ("bb", int)])])]
        control = np.array([(1.0, (1, (2, 3.0))), (2, (4, (5, 6.0)))], dtype=controldtype)
        assert_equal(test, control)

    def test_w_shorter_flex(self):
        # Test merge_arrays w/ a shorter flexndarray.
        z = self.data[-1]

        # Fixme, this test looks incomplete and broken
        # test = merge_arrays((z, np.array([10, 20, 30]).view([('C', int)])))
        # control = np.array([('A', 1., 10), ('B', 2., 20), ('-1', -1, 20)],
        #                   dtype=[('A', '|S3'), ('B', float), ('C', int)])
        # assert_equal(test, control)

        # Hack to avoid pyflakes warnings about unused variables
        merge_arrays((z, np.array([10, 20, 30]).view([("C", int)])))
        np.array([("A", 1.0, 10), ("B", 2.0, 20), ("-1", -1, 20)], dtype=[("A", "|S3"), ("B", float), ("C", int)])

    def test_flatten(self):
        # Test standard & flexible
        (_, x, _, z) = self.data
        test = merge_arrays((x, z), flatten=True)
        control = np.array([(1, 'A', 1.), (2, 'B', 2.)],
                           dtype=[('f0', int), ('A', '|S3'), ('B', float)])
        assert_equal(test, control)

        test = merge_arrays((x, z), flatten=False)
        control = np.array([(1, ('A', 1.)), (2, ('B', 2.))],
                           dtype=[('f0', int),
                                  ('f1', [('A', '|S3'), ('B', float)])])
        assert_equal(test, control)

    def test_standard(self):
        # Test standard & standard
        # Test merge arrays
        (_, x, y, _) = self.data
        test = merge_arrays((x, y), usemask=False)
        control = np.array([(1, 10), (2, 20), (-1, 30)], dtype=[("f0", int), ("f1", int)])
        assert_equal(test, control)

        test = merge_arrays((x, y), usemask=True)
        control = ma.array(
            [(1, 10), (2, 20), (-1, 30)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("f0", int), ("f1", int)]
        )
        assert_equal(test, control)
        assert_equal(test.mask, control.mask)

    def test_solo(self):
        # Test merge_arrays on a single array.
        (_, x, _, z) = self.data

        test = merge_arrays(x)
        control = np.array([(1,), (2,)], dtype=[('f0', int)])
        assert_equal(test, control)
        test = merge_arrays((x,))
        assert_equal(test, control)

        test = merge_arrays(z, flatten=False)
        assert_equal(test, z)
        test = merge_arrays(z, flatten=True)
        assert_equal(test, z)

 def test_w_singlefield(self):
     # Test single field
     test = merge_arrays((np.array([1, 2]).view([("a", int)]), np.array([10.0, 20.0, 30.0])))
     control = ma.array(
         [(1, 10.0), (2, 20.0), (-1, 30.0)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("a", int), ("f1", float)]
     )
     assert_equal(test, control)

  def chromosome_index(self):
    snps     = self.snps
    n        = len(snps)
    index    = np.arange(n).astype([('index',int)])
    chroms   = snps['chromosome'].astype('S10')
    chromset = set(chroms)
    chroms   = chroms.astype([('chromosome','S10')])
    locs     = snps['location'].astype(int).astype([('location',int)])
    snps     = rfn.merge_arrays([chroms,locs,index])

    snps.sort()

    index = {}
    for name in chromset:
      mask        = snps['chromosome']==name
      indices     = snps['index'][mask]
      pos         = snps['location'][mask]

      if name.startswith('chr'):
        name = name[3:]
      if name.upper()=='MT':
        name = 'M'

      index[name] = pos,indices

    return index

    def restrict(self, restriction):
        restriction = np.asarray(restriction, bool)
        base = rsgame.empty_copy(self).restrict(restriction)

        size_mask = restriction.repeat(self._sizes)
        sizes = self._sizes[restriction]
        profiles = self._profiles[size_mask]
        lengths = self._lengths[restriction]
        zeros = (profiles[:, ~restriction] /
                 lengths[:, ~restriction].repeat(sizes, 0))
        removed = np.exp(-np.einsum('ij,ij->i', zeros, zeros) / 2) # pylint: disable=invalid-unary-operand-type
        uprofs, inds = np.unique(
            recfunctions.merge_arrays([
                np.arange(restriction.sum()).repeat(sizes).view([('s', int)]),
                utils.axis_to_elem(profiles[:, restriction])], flatten=True),
            return_inverse=True)
        new_alpha = np.bincount(inds, removed * self._alpha[size_mask])
        new_sizes = np.diff(np.concatenate([
            [-1], np.flatnonzero(np.diff(uprofs['s'])),
            [new_alpha.size - 1]]))

        return _RbfGpGame(
            base.role_names, base.strat_names, base.num_role_players,
            self._offset[restriction], self._coefs[restriction],
            lengths[:, restriction], new_sizes, uprofs['axis'], new_alpha)

 def run(self, outputs_requested, **kwargs):
     arrays = [kwargs[input_key].to_np() for input_key in 
               self.__input_keys]
     # http://stackoverflow.com/questions/15815854/how-to-add-column-to-numpy-array
     out = UObject(UObjectPhase.Write)
     out.from_np(merge_arrays(arrays, flatten=True))
     return {'output': out}

 def test_singlerecord(self):
     (_, x, y, z) = self.data
     test = merge_arrays((x[0], y[0], z[0]), usemask=False)
     control = np.array([(1, 10, ('A', 1))],
                        dtype=[('f0', int),
                               ('f1', int),
                               ('f2', [('A', '|S3'), ('B', float)])])
     assert_equal(test, control)

def fix_hj_fracgood_colmns(objects):
    nrows = objects.size
    dtypes = np.dtype([('FRACGOOD', '>f4')])
    dummy = np.empty(nrows, dtype=dtypes)
    dummy[:] = 1.0
    obj = rfn.merge_arrays([objects,dummy], flatten=True, usemask=False)
    #
    return obj

 def test_w_singlefield(self):
     # Test single field
     test = merge_arrays((np.array([1, 2]).view([('a', int)]),
                          np.array([10., 20., 30.])),)
     control = ma.array([(1, 10.), (2, 20.), (-1, 30.)],
                        mask=[(0, 0), (0, 0), (1, 0)],
                        dtype=[('a', int), ('f1', float)])
     assert_equal(test, control)

    def finalizeObjects(self, objects):
        objs = numpy.recarray(len(objects),
                              dtype=[('NAME','S24'),
                                     ('TS','f4'),
                                     ('GLON','f4'),
                                     ('GLAT','f4'),
                                     ('RA','f4'),
                                     ('DEC','f4'),
                                     ('MODULUS','f4'),
                                     ('DISTANCE','f4'),
                                     ('RICHNESS','f4'),
                                     ('MASS','f4'),
                                     ('NANNULUS','i4'),
                                     ('NINTERIOR','i4'),
                                     ])
        
        objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']]
        lon,lat = objects['X_MAX'],objects['Y_MAX']

        coordsys = self.config['coords']['coordsys']
        if coordsys.lower() == 'gal':
            print("GAL coordintes")
            objs['GLON'],objs['GLAT'] = lon,lat
            objs['RA'],objs['DEC'] = gal2cel(lon,lat)
        else:
            print("CEL coordintes")
            objs['RA'],objs['DEC'] = lon,lat
            objs['GLON'],objs['GLAT'] = cel2gal(lon,lat)

        modulus = objects['Z_MAX']
        objs['MODULUS'] = modulus
        objs['DISTANCE'] = mod2dist(modulus)

        nside = healpy.npix2nside(len(self.nannulus))
        pix = ang2pix(nside,lon,lat)

        richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']]
        objs['RICHNESS'] = richness
        objs['MASS'] = richness * self.stellar[pix]

        objs['NANNULUS']  = self.nannulus[pix].astype(int)
        objs['NINTERIOR'] = self.ninterior[pix].astype(int)

        # Default name formatting
        # http://cdsarc.u-strasbg.fr/ftp/pub/iau/
        # http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
        fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
        for obj,_ra,_dec in zip(objs,objs['RA'],objs['DEC']):
            hms = dec2hms(_ra); dms = dec2dms(_dec)
            params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60.,
                          deg=dms[0],dmin=dms[1]+dms[2]/60.)
            obj['NAME'] = fmt%params

        out = recfuncs.merge_arrays([objs,objects],usemask=False,
                                    asrecarray=True,flatten=True)

        return out

    def finalizeObjects(self, objects):
        objs = numpy.recarray(len(objects),
                              dtype=[('NAME','S24'),
                                     ('TS','f4'),
                                     ('GLON','f4'),
                                     ('GLAT','f4'),
                                     ('RA','f4'),
                                     ('DEC','f4'),
                                     ('MODULUS','f4'),
                                     ('DISTANCE','f4'),
                                     ('RICHNESS','f4'),
                                     ('MASS','f4'),
                                     ('NANNULUS','i4'),
                                     ('NINTERIOR','i4'),
                                     ])
        
        objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']]

        glon,glat = objects['X_MAX'],objects['Y_MAX']
        objs['GLON'],objs['GLAT'] = glon,glat
        
        ra,dec    = gal2cel(glon,glat)
        objs['RA'],objs['DEC'] = ra,dec

        modulus = objects['Z_MAX']
        objs['MODULUS'] = modulus
        objs['DISTANCE'] = mod2dist(modulus)

        #ninterior = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NINSIDE')
        #nannulus = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NANNULUS')
        #stellar = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'STELLAR')

        nside = healpy.npix2nside(len(self.nannulus))
        pix = ang2pix(nside,glon,glat)

        richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']]
        objs['RICHNESS'] = richness
        objs['MASS'] = richness * self.stellar[pix]

        objs['NANNULUS']  = self.nannulus[pix].astype(int)
        objs['NINTERIOR'] = self.ninterior[pix].astype(int)

        # Default name formatting
        # http://cdsarc.u-strasbg.fr/ftp/pub/iau/
        # http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
        fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
        for obj,_ra,_dec in zip(objs,ra,dec):
            hms = dec2hms(_ra); dms = dec2dms(_dec)
            params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60.,
                          deg=dms[0],dmin=dms[1]+dms[2]/60.)
            obj['NAME'] = fmt%params

        out = recfuncs.merge_arrays([objs,objects],usemask=False,asrecarray=True,flatten=True)
        # This is safer than viewing as FITS_rec
        return pyfits.new_table(out).data

    def test_sql(self):

        # Make sure we don't accidentally corrupt our test database
        db_path, db_file_name = self._tmp_files.tmp_copy(path_of_data(
            'small.db'))
        db_url = 'sqlite:///{}'.format(db_path)
        
        q_sel_employees = 'CREATE TABLE {tmp_emp} AS SELECT * FROM employees;'
        # We have to be careful about the datetime type in sqlite3. It will
        # forget if we don't keep reminding it, and if it forgets sqlalchemy
        # will be unhappy. Hence, we can't use CREATE TABLE AS if our table
        # has a DATETIME
        q_sel_hours = ('CREATE TABLE {tmp_hrs} '
                       '(id INT, employee_id INT, time DATETIME, '
                       '    event_type TEXT); '
                       'INSERT INTO {tmp_hrs} SELECT * FROM hours;')
        q_join = ('CREATE TABLE {joined} '
                  '(id INT, last_name TEXT, salary REAL, time DATETIME, '
                  '    event_type TEXT); '
                  'INSERT INTO {joined} '
                  'SELECT {tmp_emp}.id, last_name, salary, time, event_type '
                  'FROM {tmp_emp} JOIN {tmp_hrs} ON '
                  '{tmp_emp}.id = {tmp_hrs}.employee_id;')

        p = Pipeline()
        get_emp = p.add(RunSQL(db_url, q_sel_employees, [], ['tmp_emp'], {}))
        get_hrs = p.add(RunSQL(db_url, q_sel_hours, [], ['tmp_hrs'], {}))
        join = p.add(RunSQL(db_url, q_join, ['tmp_emp', 'tmp_hrs'], ['joined'],
                            {}))
        csv_out = p.add(CSVWrite(self._tmp_files('out.csv')))

        get_emp['tmp_emp'] > join['tmp_emp']
        get_hrs['tmp_hrs'] > join['tmp_hrs']
        join['joined'] > csv_out['input']

        self.run_pipeline(p)

        ctrl = csv_read(path_of_data('test_transform_test_sql_ctrl.csv'))
        result = self._tmp_files.csv_read('out.csv')
        # Because Numpy insists on printing times with local offsets, but
        # not every computer has the same offset, we have to force it back
        # into UTC
        for i, dt in enumerate(result['time']):
            # .item() makes a datetime, which we can format correctly later
            # http://stackoverflow.com/questions/25134639/how-to-force-python-print-numpy-datetime64-with-specified-timezone
            result['time'][i] = np.datetime64(dt).item().strftime(
                    '%Y-%m-%dT%H:%M:%S')
        # Then we have to make the string field smaller
        new_cols = []
        for col in result.dtype.names:
            new_cols.append(result[col].astype(ctrl.dtype[col]))
        result = merge_arrays(new_cols, flatten=True) 
        result.dtype.names = ctrl.dtype.names

        self.assertTrue(np.array_equal(result, ctrl))

def classify_line(filename, classifier):
    """ Use `classifier` to classify data stored in `filename`

    Args:
      filename (str): filename of stored results
      classifier (sklearn classifier): pre-trained classifier

    """
    z = np.load(filename)
    rec = z['record']

    if rec.shape[0] == 0:
        logger.debug('No records in {f}. Continuing'.format(f=filename))
        return

    # Rescale intercept term
    coef = rec['coef'].copy()  # copy so we don't transform npz coef
    coef[:, 0, :] = (coef[:, 0, :] + coef[:, 1, :] *
                     ((rec['start'] + rec['end']) / 2.0)[:, np.newaxis])

    # Include RMSE for full X matrix
    newdim = (coef.shape[0], coef.shape[1] * coef.shape[2])
    X = np.hstack((coef.reshape(newdim), rec['rmse']))

    # Create output and classify
    classes = classifier.classes_
    classified = np.zeros(rec.shape[0], dtype=[
        ('class', 'u2'),
        ('class_proba', 'float32', classes.size)
    ])
    classified['class'] = classifier.predict(X)
    classified['class_proba'] = classifier.predict_proba(X)

    # Replace with new classification if exists, or add by merging
    if ('class' in rec.dtype.names and 'class_proba' in rec.dtype.names and
            rec['class_proba'].shape[1] == classes.size):
        rec['class'] = classified['class']
        rec['class_proba'] = classified['class_proba']
    else:
        # Drop incompatible classified results if needed
        # e.g., if the number of classes changed
        if 'class' in rec.dtype.names and 'class_proba' in rec.dtype.names:
            rec = nprfn.drop_fields(rec, ['class', 'class_proba'])
        rec = nprfn.merge_arrays((rec, classified), flatten=True)

    # Create dict for re-saving `npz` file (only way to append)
    out = {}
    for k, v in z.iteritems():
        out[k] = v
    out['classes'] = classes
    out['record'] = rec

    np.savez(filename, **out)