def test_generate_examples_1_channel():
    """Run a flat epoch and an active epoch of learning, compare learned sets.

    The *order* of the edges learned by learn_flat is not guaranteed, so we
    test the *set* of learned edges for the flat epoch. The learned epoch
    *should* have a fixed order, so we test array equality.

    Uses 1 channel probabilities.
    """
    g_train = agglo.Rag(ws_train, pr_train, feature_manager=fc)
    _, alldata = g_train.learn_agglomerate(gt_train, fc, classifier="naive bayes")
    testfn = (
        "example-data/train-naive-bayes-merges1-py3.pck"
        if PYTHON_VERSION == 3
        else "example-data/train-naive-bayes-merges1-py2.pck"
    )
    exp0, exp1 = load_pickle(os.path.join(rundir, testfn))
    expected_edges = set(map(tuple, exp0))
    edges = set(map(tuple, alldata[0][3]))
    merges = alldata[1][3]
    assert edges == expected_edges
    # concordant is the maximum edges concordant in the Python 2.7 version.
    # The remaining edges diverge because of apparent differences
    # between Linux and OSX floating point handling.
    concordant = slice(None, 171) if PYTHON_VERSION == 2 else slice(None)
    assert_array_equal(merges[concordant], exp1[concordant])
    nb = GaussianNB().fit(alldata[0][0], alldata[0][1][:, 0])
    nbexp = joblib.load(os.path.join(rundir, "example-data/naive-bayes-1.joblib"))
    assert_allclose(nb.theta_, nbexp.theta_, atol=1e-10)
    assert_allclose(nb.sigma_, nbexp.sigma_, atol=1e-4)
    assert_allclose(nb.class_prior_, nbexp.class_prior_, atol=1e-7)

    def test_xform_parsing_with_stock_questions(self):
        form_xml = self.get_xml('stock_form')
        schema = FormExportDataSchema._generate_schema_from_xform(
            XForm(form_xml),
            ['en'],
            self.app_id,
            1
        )
        self.assertEqual(len(schema.group_schemas), 1)
        group_schema = schema.group_schemas[0]

        self.assertEqual(len(group_schema.items), 6)
        self.assertTrue(all([item.doc_type == 'StockItem' for item in group_schema.items]))
        for parent_attr in ['@type', '@entity-id', '@date', '@section-id']:
            self.assertTrue(any(map(
                lambda item: item.path == [
                    PathNode(name='form'),
                    PathNode(name='balance:balance_one'),
                    PathNode(name=parent_attr),
                ],
                group_schema.items,
            )))

        for entry_attr in ['@id', '@quantity']:
            self.assertTrue(any(map(
                lambda item: item.path == [
                    PathNode(name='form'),
                    PathNode(name='balance:balance_one'),
                    PathNode(name='entry'),
                    PathNode(name=entry_attr),
                ],
                group_schema.items,
            )))

    def get_revert_migrations(self, current_migrations, backup_migrations):
        current_migrations, all_migrations = itertools.tee(reversed(list(map(
            Migration,
            filter(None, current_migrations.splitlines()),
        ))))
        all_migrations = utils.OrderedSet(all_migrations)

        backup_migrations = reversed(list(map(
            Migration,
            filter(None, backup_migrations.splitlines()),
        )))

        revert_migrations = collections.OrderedDict()

        while True:
            while True:
                backup_migration = next(backup_migrations, None)
                if not backup_migration or backup_migration in all_migrations:
                    break
            for current_migration in current_migrations:
                if current_migration == backup_migration:
                    break
                revert_migration = self._get_parent_migration(
                    current_migration,
                    migrations=all_migrations,
                )
                revert_migrations[current_migration.app] = revert_migration

            if backup_migration is None:
                return revert_migrations.values()

def _get_plot_defn(cube, mode, ndims=2):
    """
    Return data and plot-axis coords given a cube & a mode of either
    POINT_MODE or BOUND_MODE.

    """
    if cube.ndim != ndims:
        msg = 'Cube must be %s-dimensional. Got %s dimensions.'
        raise ValueError(msg % (ndims, cube.ndim))

    # Start by taking the DimCoords from each dimension.
    coords = [None] * ndims
    for dim_coord in cube.dim_coords:
        dim = cube.coord_dims(dim_coord)[0]
        coords[dim] = dim_coord

    # When appropriate, restrict to 1D with bounds.
    if mode == iris.coords.BOUND_MODE:
        coords = list(map(_valid_bound_coord, coords))

    def guess_axis(coord):
        axis = None
        if coord is not None:
            axis = iris.util.guess_coord_axis(coord)
        return axis

    # Allow DimCoords in aux_coords to fill in for missing dim_coords.
    for dim, coord in enumerate(coords):
        if coord is None:
            aux_coords = cube.coords(dimensions=dim)
            aux_coords = [coord for coord in aux_coords
                          if isinstance(coord, iris.coords.DimCoord)]
            if aux_coords:
                key_func = lambda coord: coord._as_defn()
                aux_coords.sort(key=key_func)
                coords[dim] = aux_coords[0]

    if mode == iris.coords.POINT_MODE:
        # Allow multi-dimensional aux_coords to override the dim_coords
        # along the Z axis. This results in a preference for using the
        # derived altitude over model_level_number or level_height.
        # Limit to Z axis to avoid preferring latitude over grid_latitude etc.
        axes = list(map(guess_axis, coords))
        axis = 'Z'
        if axis in axes:
            for coord in cube.coords(dim_coords=False):
                if max(coord.shape) > 1 and \
                        iris.util.guess_coord_axis(coord) == axis:
                    coords[axes.index(axis)] = coord

    # Re-order the coordinates to achieve the preferred
    # horizontal/vertical associations.
    def sort_key(coord):
        order = {'X': 2, 'T': 1, 'Y': -1, 'Z': -2}
        axis = guess_axis(coord)
        return (order.get(axis, 0), coord and coord.name())
    sorted_coords = sorted(coords, key=sort_key)

    transpose = (sorted_coords != coords)
    return PlotDefn(sorted_coords, transpose)

    def get_network_create_endpoint_kwargs(self, action, endpoint_config, kwargs=None):
        """
        Generates keyword arguments for Docker's ``create_endpoint_config`` utility / ``EndpointConfig`` type as well
        as for ``connect_container_to_network``.

        :param action: Action configuration.
        :type action: ActionConfig
        :param endpoint_config: Network endpoint configuration.
        :type endpoint_config: dockermap.map.input.NetworkEndpoint
        :param kwargs: Additional keyword arguments to complement or override the configuration-based values.
        :type kwargs: dict
        :return: Resulting keyword arguments.
        :rtype: dict
        """
        map_name = action.config_id.map_name
        policy = self._policy
        c_kwargs = dict(
            ipv4_address=resolve_value(endpoint_config.ipv4_address),
            ipv6_address=resolve_value(endpoint_config.ipv6_address),
        )
        if endpoint_config.aliases:
            c_kwargs['aliases'] = list(map(resolve_value, endpoint_config.aliases))
        if endpoint_config.links:
            c_kwargs['links'] = [(policy.cname(map_name, l_name), alias or policy.get_hostname(l_name))
                                 for l_name, alias in endpoint_config.links]
        if endpoint_config.link_local_ips:
            c_kwargs['link_local_ips'] = list(map(resolve_value, endpoint_config.link_local_ips))
        update_kwargs(c_kwargs, kwargs)
        return c_kwargs

def normboolclass(typename='NormalizedBool', true=None, false=None,
                 ignore='', caseless=True, spaceless=True,
                 base=NormalizedBool):

    if not issubclass(base, NormalizedBool):
        raise TypeError("'base' is no subclass of normboolclass.base: %s"
                        % base)

    # to be stored as .normalize method of created class
    def normalizer(value):
        """Normalize `value` based on normalizing options
           given to :func:`normboolclass`.

        - Any non-string values are just passed through.
        """
        if not isinstance(value, string_types):
            return value
        return normalize(value, ignore=normalizer.ignore,
          caseless=normalizer.caseless, spaceless=normalizer.spaceless)

    # store the normalizing options
    normalizer.ignore = ignore
    normalizer.caseless = caseless
    normalizer.spaceless = spaceless

    if true:
        true = list(map(normalizer, true))
    if false:
        false = list(map(normalizer, false))

    Bool = boolclass(typename, true=true, false=false, base=base)
    type(Bool).normalize = staticmethod(normalizer)
    return Bool

 def disassemble(cls, payload):
     (is_stored_as_bytes, user_id) = payload[0]
     if is_stored_as_bytes:
         user_id = cls.convert_uuid_bytes_to_hex(user_id)
     methods = auth_plugins.convert_integer_to_method_list(payload[1])
     (is_stored_as_bytes, scope_id) = payload[2]
     if is_stored_as_bytes:
         scope_id = cls.convert_uuid_bytes_to_hex(scope_id)
     project_id = (
         scope_id
         if cls.version == FederatedProjectScopedPayload.version else None)
     domain_id = (
         scope_id
         if cls.version == FederatedDomainScopedPayload.version else None)
     group_ids = list(map(cls.unpack_group_id, payload[3]))
     (is_stored_as_bytes, idp_id) = payload[4]
     if is_stored_as_bytes:
         idp_id = cls.convert_uuid_bytes_to_hex(idp_id)
     protocol_id = payload[5]
     expires_at_str = cls._convert_float_to_time_string(payload[6])
     audit_ids = list(map(provider.base64_encode, payload[7]))
     federated_info = dict(idp_id=idp_id, protocol_id=protocol_id,
                           group_ids=group_ids)
     trust_id = None
     access_token_id = None
     return (user_id, methods, project_id, domain_id, expires_at_str,
             audit_ids, trust_id, federated_info, access_token_id)

    def print_kernel_info(self, output_file=sys.stdout):
        table = ('     idx |        min        max       step\n' +
                 '---------+---------------------------------\n')
        for l in self._loop_stack:
            table += '{:>8} | {!s:>10} {!s:>10} {:>+10}\n'.format(*l)
        print(prefix_indent('loop stack:        ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self._sources.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join([', '.join(map(tuple.__repr__, o)) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data sources:      ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self._destinations.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join([', '.join(map(tuple.__repr__, o)) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data destinations: ', table), file=output_file)

        table = (' op | count \n' +
                 '----+-------\n')
        for op, count in list(self._flops.items()):
            table += '{:>3} | {:>4}\n'.format(op, count)
        table += '     =======\n'
        table += '      {:>4}'.format(sum(self._flops.values()))
        print(prefix_indent('FLOPs:     ', table), file=output_file)

 def disassemble(cls, payload):
     (is_stored_as_bytes, user_id) = payload[0]
     if is_stored_as_bytes:
         user_id = cls.convert_uuid_bytes_to_hex(user_id)
     methods = auth_plugins.convert_integer_to_method_list(payload[1])
     group_ids = list(map(cls.unpack_group_id, payload[2]))
     (is_stored_as_bytes, idp_id) = payload[3]
     if is_stored_as_bytes:
         idp_id = cls.convert_uuid_bytes_to_hex(idp_id)
     else:
         idp_id = idp_id.decode('utf-8')
     protocol_id = payload[4]
     if isinstance(protocol_id, six.binary_type):
         protocol_id = protocol_id.decode('utf-8')
     expires_at_str = cls._convert_float_to_time_string(payload[5])
     audit_ids = list(map(cls.base64_encode, payload[6]))
     federated_info = dict(group_ids=group_ids, idp_id=idp_id,
                           protocol_id=protocol_id)
     system = None
     project_id = None
     domain_id = None
     trust_id = None
     access_token_id = None
     app_cred_id = None
     return (user_id, methods, system, project_id, domain_id,
             expires_at_str, audit_ids, trust_id, federated_info,
             access_token_id, app_cred_id)

    def __call__(self, vals):
        if not self.colMajor:
            shapes = [list(map(lambda f: f(val), self.fns)) for val in vals]
        else:
            shapes = [list(map(lambda f: f(val), self.fns)) for val in zip(*vals)]

        return shapes

def sub_miller(segments):
    '''
    Calculates Miller indices from segments.

    Algorithm:

    1. Obtain inverted fraction from segments

    2. Find common denominator of inverted fractions

    3. Lead fractions to common denominator and throws denominator away.

    4. Return obtained values.

    Args:
        List of 3 floats, meaning distances that plane cuts on x, y, z axes.
        Any float not equals zero, it means that plane does not intersect origin,
        i. e. shift of origin has already been done.

    Returns:
        String that represents Miller indices, e.g: (-6,3,-6) or (2,2,2)
    '''
    fracts = [segment_to_fraction(segment) for segment in segments]
    common_denominator = reduce(lcm, [fract.denominator for fract in fracts])
    miller_indices = ([
        fract.numerator * math.fabs(common_denominator) / fract.denominator
        for fract in fracts
    ])
    return'(' + ','.join(map(str, list(map(decimal.Decimal, miller_indices)))) + ')'

def augment_uuid(uuid_, *hashables):
    #from six.moves import reprlib
    #uuidhex_data   = uuid_.get_bytes()
    uuidhex_data   = uuid_.bytes
    #hashable_str    = ''.join(map(repr, hashables))
    # Python 2 and 3 diverge here because repr returns
    # ascii data in python2 and unicode text in python3
    # it would be nice to
    warnings.warn('[ut] should not use repr when hashing', RuntimeWarning)
    def tmprepr(x):
        y = repr(x)
        # hack to remove u prefix
        if isinstance(x, six.string_types):
            if y.startswith('u'):
                y = y[1:]
        return y
    if six.PY2:
        hashable_text = ''.join(map(tmprepr, hashables))
        hashable_data = hashable_text.encode('utf-8')
        #hashable_data = b''.join(map(bytes, hashables))
    elif six.PY3:
        hashable_text    = ''.join(map(tmprepr, hashables))
        hashable_data = hashable_text.encode('utf-8')
        #hashable_data = b''.join(map(bytes, hashables))
    augmented_data   = uuidhex_data + hashable_data
    augmented_uuid_ = hashable_to_uuid(augmented_data)
    return augmented_uuid_

    def dump_power_calibration(self, coefficients, calibration_path=None):

        #        calibration_path = self._get_calibration_path(calibration_path)
        #        self.info('dumping power calibration {}'.format(calibration_path))

        coeffstr = lambda c: 'calibration coefficients= {}'.format(', '.join(map('{:0.3e}'.format, c)))
        self.info(coeffstr(coefficients))
        #        if bounds:
        #            for coeffs, bi in zip(coefficients, bounds):
        #                self.info('calibration coefficient')
        #            self.info('{} min={:0.2f}, max={:0.2f}'.format(coeffstr(coeffs, *bi)))
        #        else:
        #            self.info(coeffstr(coefficients))
        #
        #        pc = MeterCalibration(coefficients)
        #        pc.bounds = bounds
        #        try:
        #            with open(calibration_path, 'wb') as f:
        #                pickle.dump(pc, f)
        #        except  (pickle.PickleError, EOFError, OSError), e:
        #            self.warning('pickling error {}'.format(e))

        # also update logic board configuration file
        if self.parent is not None:
            lb = self.parent.laser_controller
            config = lb.get_configuration()
            section = 'PowerOutput'
            if not config.has_section(section):
                config.add_section(section)

            config.set(section,
                       'coefficients',
                       ','.join(map('{:0.3e}'.format, coefficients))
                       )
            lb.write_configuration(config)

def list_entities():
    """Return a JSON object with a list of all of the specified entity type.

    Example output::
        {
          entities: [
            {name: 'Bar', id: 1, source: 'Auth Source', type: 'User'},
            {name: 'Foo', id: 0, source: 'Another Auth Source', type: 'Group'},
            {name: 'Baz', id: 20, source: 'Auth Source', type: 'Group'}
          ]
        }

    This method is only accesible to administrators.

    :param str entity_type: Either ``'user'`` or ``'group'``.
    """
    if not current_user.admin and not \
            current_user.has_permission(PermissionType.admin):
        abort(403)
    user_query = db.session.query(User.id, User.name, User.authmethod)
    group_query = db.session.query(Group.id, Group.name, Group.authmethod)
    users = map(lambda e: {
            u'id': e.id,
            u'name': e.name,
            u'type': u'User',
            u'source': e.authmethod}, user_query)
    groups = map(lambda e: {
            u'id': e.id,
            u'name': e.name,
            u'type': u'Group',
            u'source': e.authmethod}, group_query)
    return jsonify(entities=chain(users, groups))

 def _json(self, extended=False):
     try:
         parent = super(Request, self)._json(extended)
     except AttributeError:
         parent = {}
     parent[u'href'] = url_for('requests.get_request_details',
             request_id=self.id)
     attrs = (u'killmail_url', u'kill_timestamp', u'pilot',
              u'alliance', u'corporation', u'submitter',
              u'division', u'status', u'base_payout', u'payout',
              u'details', u'id', u'ship_type', u'system', u'constellation',
              u'region')
     for attr in attrs:
         if attr == u'ship_type':
             parent['ship'] = self.ship_type
         elif u'payout' in attr:
             payout = getattr(self, attr)
             parent[attr] = payout.currency()
         else:
             parent[attr] = getattr(self, attr)
     parent[u'submit_timestamp'] = self.timestamp
     if extended:
         parent[u'actions'] = map(lambda a: a._json(True), self.actions)
         parent[u'modifiers'] = map(lambda m: m._json(True), self.modifiers)
         parent[u'valid_actions'] = self.valid_actions(current_user)
         parent[u'transformed'] = dict(self.transformed)
     return parent

 def _from_word2vec_text(fname):
   with _open(fname, 'rb') as fin:
     words = []
     header = unicode(fin.readline())
     vocab_size, layer1_size = list(map(int, header.split())) # throws for invalid file format
     vectors = []
     for line_no, line in enumerate(fin):
       try:
         parts = unicode(line, encoding="utf-8").strip().split()
       except TypeError as e:
         parts = line.strip().split()
       except Exception as e:
         logger.warning("We ignored line number {} because of erros in parsing"
                         "\n{}".format(line_no, e))
         continue
       # We differ from Gensim implementation.
       # Our assumption that a difference of one happens because of having a
       # space in the word.
       if len(parts) == layer1_size + 1:
         word, weights = parts[0], list(map(float32, parts[1:]))
       elif len(parts) == layer1_size + 2:
         word, weights = parts[:2], list(map(float32, parts[2:]))
         word = u" ".join(word)
       else:
         logger.warning("We ignored line number {} because of unrecognized "
                         "number of columns {}".format(line_no, parts[:-layer1_size]))
         continue
       index = line_no
       words.append(word)
       vectors.append(weights)
     vectors = np.asarray(vectors, dtype=np.float32)
     return words, vectors

 def disassemble(cls, payload):
     (is_stored_as_bytes, user_id) = payload[0]
     if is_stored_as_bytes:
         user_id = cls.convert_uuid_bytes_to_hex(user_id)
     else:
         # NOTE(cmurphy): The user ID of shadowed federated users is no
         # longer a UUID but a sha256 hash string, and so it should not be
         # converted to a byte string since it is not a UUID format.
         # However. on python3 msgpack returns the serialized input as a
         # byte string anyway. Similar to other msgpack'd values in the
         # payload, we need to explicitly decode it to a string value.
         if six.PY3 and isinstance(user_id, six.binary_type):
             user_id = user_id.decode('utf-8')
     methods = auth_plugins.convert_integer_to_method_list(payload[1])
     group_ids = list(map(cls.unpack_group_id, payload[2]))
     (is_stored_as_bytes, idp_id) = payload[3]
     if is_stored_as_bytes:
         idp_id = cls.convert_uuid_bytes_to_hex(idp_id)
     else:
         idp_id = idp_id.decode('utf-8')
     protocol_id = payload[4]
     if isinstance(protocol_id, six.binary_type):
         protocol_id = protocol_id.decode('utf-8')
     expires_at_str = cls._convert_float_to_time_string(payload[5])
     audit_ids = list(map(cls.base64_encode, payload[6]))
     system = None
     project_id = None
     domain_id = None
     trust_id = None
     access_token_id = None
     app_cred_id = None
     return (user_id, methods, system, project_id, domain_id,
             expires_at_str, audit_ids, trust_id, group_ids, idp_id,
             protocol_id, access_token_id, app_cred_id)

def build_clause(query, attributes):
    # Iterate through each query segment.
    clause = None
    last = None
    for seg in query.segments:
        # Get the attribute in question.
        attribute = attributes[seg.path[0]]

        # Replace the initial path segment with the expanded
        # attribute path.
        seg.path[0:1] = attribute.path.split('.')

        # Boolean's should use `exact` rather than `iexact`.
        if attribute.type is bool:
            op = '__exact'
        else:
            op = OPERATOR_MAP[seg.operator]

        # Build the path from the segment.
        path = '__'.join(seg.path) + op

        # Construct a Q-object from the segment.
        q = reduce(operator.or_,
                   map(lambda x: Q((path, x)),
                       map(attribute.try_clean, seg.values)))

        # Combine the segment with the last.
        clause = last.combinator(clause, q) if last is not None else q
        last = seg

    # Return the constructed clause.
    return clause

def check(process_output, judge_output, split_on='lines', **kwargs):
    split_pattern = {
        'lines': b'[\r\n]',
        'whitespace': b'[\s]',
    }.get(split_on)

    if not split_pattern:
        raise InternalError('invalid `split_on` mode')

    process_lines = list(filter(None, resplit(split_pattern, utf8bytes(process_output))))
    judge_lines = list(filter(None, resplit(split_pattern, utf8bytes(judge_output))))

    if len(process_lines) != len(judge_lines):
        return False

    if split_on == 'lines':
        process_lines = list(map(six.binary_type.split, process_lines))
        judge_lines = list(map(six.binary_type.split, judge_lines))

    process_lines.sort()
    judge_lines.sort()

    for process_line, judge_line in zip(process_lines, judge_lines):
        if process_line != judge_line:
            return False

    return True

def _get_features_string(self, features=None):
    """ Generates the extended newick string NHX with extra data about
    a node. """
    string = ""
    if features is None:
        features = []
    elif features == []:
        features = self.features

    for pr in features:
        if hasattr(self, pr):
            raw = getattr(self, pr)
            if type(raw) in ITERABLE_TYPES:
                raw = '|'.join(map(str, raw))
            elif type(raw) == dict:
                raw = '|'.join(map(lambda x,y: "%s-%s" %(x, y), six.iteritems(raw)))
            elif type(raw) == str:
                pass
            else:
                raw = str(raw)

            value = re.sub("["+_ILEGAL_NEWICK_CHARS+"]", "_", \
                             raw)
            if string != "":
                string +=":"
            string +="%s=%s"  %(pr, str(value))
    if string != "":
        string = "[&&NHX:"+string+"]"

    return string