def record_derivatives(self, derivs, metadata):
        """Writes the derivatives that were calculated for the driver.

        Args
        ----
        derivs : dict
            Dictionary containing derivatives

        metadata : dict, optional
            Dictionary containing execution metadata (e.g. iteration coordinate).
        """

        iteration_coordinate = metadata['coord']
        timestamp = metadata['timestamp']

        write = self.out.write
        fmat = "Timestamp: {0!r}\n"
        write(fmat.format(timestamp))

        fmat = "Iteration Coordinate: {0:s}/Derivs\n"
        write(fmat.format(format_iteration_coordinate(iteration_coordinate)))

        write("Derivatives:\n")
        if isinstance(derivs, dict):
            for okey, sub in sorted(iteritems(derivs)):
                for ikey, deriv in sorted(iteritems(sub)):
                    write("  {0} wrt {1}: {2}\n".format(okey, ikey, str(deriv)))
        else:
            write("  {0} \n".format(str(derivs)))

        # Flush once per iteration to allow external scripts to process the data.
        self.out.flush()

    def test_driver_records_unknown_types_metadata(self):

        prob = Problem()
        root = prob.root = Group()

        # Need an optimization problem to test to make sure
        #   the is_desvar, is_con, is_obj metadata is being
        #   recorded for the Unknowns
        root.add('p1', IndepVarComp('x', 50.0))
        root.add('p2', IndepVarComp('y', 50.0))
        root.add('comp', Paraboloid())

        root.connect('p1.x', 'comp.x')
        root.connect('p2.y', 'comp.y')

        prob.driver = ScipyOptimizer()
        prob.driver.options['optimizer'] = 'SLSQP'
        prob.driver.add_desvar('p1.x', lower=-50.0, upper=50.0)
        prob.driver.add_desvar('p2.y', lower=-50.0, upper=50.0)

        prob.driver.add_objective('comp.f_xy')
        prob.driver.options['disp'] = False

        prob.driver.add_recorder(self.recorder)
        self.recorder.options['record_metadata'] = True
        prob.setup(check=False)
        prob.cleanup() # close recorders

        expected_params = list(iteritems(prob.root.params))
        expected_unknowns = list(iteritems(prob.root.unknowns))
        expected_resids = list(iteritems(prob.root.resids))

        self.assertMetadataRecorded((expected_params, expected_unknowns, expected_resids))

    def __call_api(self, resource_path, method,
                   path_params=None, query_params=None, header_params=None,
                   body=None, post_params=None, files=None,
                   response_type=None, auth_settings=None, callback=None):

        # headers parameters
        header_params = header_params or {}
        header_params.update(self.default_headers)
        if self.cookie:
            header_params['Cookie'] = self.cookie
        if header_params:
            header_params = self.sanitize_for_serialization(header_params)

        # path parameters
        if path_params:
            path_params = self.sanitize_for_serialization(path_params)
            for k, v in iteritems(path_params):
                replacement = quote(str(self.to_path_value(v)))
                resource_path = resource_path.\
                    replace('{' + k + '}', replacement)

        # query parameters
        if query_params:
            query_params = self.sanitize_for_serialization(query_params)
            query_params = {k: self.to_path_value(v)
                            for k, v in iteritems(query_params)}

        # post parameters
        if post_params or files:
            post_params = self.prepare_post_parameters(post_params, files)
            post_params = self.sanitize_for_serialization(post_params)

        # auth setting
        self.update_params_for_auth(header_params, query_params, auth_settings)

        # body
        if body:
            body = self.sanitize_for_serialization(body)

        # request url
        url = self.host + resource_path

        # perform request and return response
        response_data = self.request(method, url,
                                     query_params=query_params,
                                     headers=header_params,
                                     post_params=post_params, body=body)

        self.last_response = response_data

        # deserialize response data
        if response_type:
            deserialized_data = self.deserialize(response_data, response_type)
        else:
            deserialized_data = None

        if callback:
            callback(deserialized_data)
        else:
            return deserialized_data

 def get_objects(self):
     # type: () -> Iterator[Tuple[unicode, unicode, unicode, unicode, unicode, int]]
     for modname, info in iteritems(self.data['modules']):
         yield (modname, modname, 'module', info[0], 'module-' + modname, 0)
     for refname, (docname, type) in iteritems(self.data['objects']):
         if type != 'module':  # modules are already handled
             yield (refname, refname, type, docname, refname, 1)

def _resolve_with_state(unconflicted_state_ids, conflicted_state_ids, auth_event_ids,
                        state_map):
    conflicted_state = {}
    for key, event_ids in iteritems(conflicted_state_ids):
        events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map]
        if len(events) > 1:
            conflicted_state[key] = events
        elif len(events) == 1:
            unconflicted_state_ids[key] = events[0].event_id

    auth_events = {
        key: state_map[ev_id]
        for key, ev_id in iteritems(auth_event_ids)
        if ev_id in state_map
    }

    try:
        resolved_state = _resolve_state_events(
            conflicted_state, auth_events
        )
    except Exception:
        logger.exception("Failed to resolve state")
        raise

    new_state = unconflicted_state_ids
    for key, event in iteritems(resolved_state):
        new_state[key] = event.event_id

    return new_state

    def __setstate__(self, state):
        """ Overwrite message state with given kwargs. """
        self._tokens = state['tokens']
        self.extra = {}
        self.expiry = state['expiry']
        self.priority = state['priority']

        if 'payload' in state:
            self._payload = state['payload']
            if hasattr(self._payload, "get") and self._payload.get("aps"):
                aps = self._payload["aps"]
                self.alert = aps.get("alert")
                self.badge = aps.get("badge")
                self.sound = aps.get("sound")
                self.content_available = aps.get("content-available")
                self.extra = dict([(k, v) for (k, v) in six.iteritems(self._payload) if k != 'aps'])
        else:
            self._payload = None
            for key, val in six.iteritems(state):
                if key in ('tokens', 'expiry', 'priority'):  # already set
                    pass
                elif key in ('alert', 'badge', 'sound', 'content_available'):
                    setattr(self, key, state[key])
                elif key == 'extra':
                    self.extra.update(state[key])
                else:
                    # legacy serialized object
                    self.extra[key] = val

 def get_data(self, parsed_args):
     self.log.debug('get_data(%s)', parsed_args)
     neutron_client = self.get_client()
     neutron_client.format = parsed_args.request_format
     tenant_id = get_tenant_id(parsed_args.tenant_id,
                               neutron_client)
     params = {}
     obj_shower = getattr(neutron_client,
                          "show_%s" % self.resource)
     data = obj_shower(tenant_id, **params)
     if self.resource in data:
         for k, v in six.iteritems(data[self.resource]):
             if isinstance(v, list):
                 value = ""
                 for _item in v:
                     if value:
                         value += "\n"
                     if isinstance(_item, dict):
                         value += jsonutils.dumps(_item)
                     else:
                         value += str(_item)
                 data[self.resource][k] = value
             elif v is None:
                 data[self.resource][k] = ''
         return zip(*sorted(six.iteritems(data[self.resource])))
     else:
         return None

    def setMetadata(self, folder, metadata):
        """
        Set metadata on a folder.  A rest exception is thrown in the cases
        where the metadata json object is badly formed, or if any of the
        metadata keys contains a period ('.').

        :param folder: The folder to set the metadata on.
        :type folder: dict
        :param metadata: A dictionary containing key-value pairs to add to
                     the folder's meta field
        :type metadata: dict
        :returns: the folder document
        """
        if 'meta' not in folder:
            folder['meta'] = {}

        # Add new metadata to existing metadata
        folder['meta'].update(six.iteritems(metadata))

        # Remove metadata fields that were set to null (use items in py3)
        folder['meta'] = {k: v
                          for k, v in six.iteritems(folder['meta'])
                          if v is not None}

        folder['updated'] = datetime.datetime.utcnow()

        # Validate and save the item
        return self.save(folder)

    def __init__(self, query_dict=None, q=None, fq=None, fl=DEFAULT_FIELDS,
                 sort=None, start=0, rows=50, max_pages=1, token=None, **kwargs):
        """
        constructor
        :param query_dict: raw query that will be sent unmodified. raw takes
            precedence over individually defined query params
        :type query_dict: dict
        :param q: solr "q" param (query)
        :param fq: solr "fq" param (filter query)
        :param fl: solr "fl" param (filter limit)
        :param sort: solr "sort" param (sort)
        :param start: solr "start" param (start)
        :param rows: solr "rows" param (rows)
        :param max_pages: Maximum number of pages to return. This value may
            be modified after instantiation to increase the number of results
        :param token: optional API token to use for this searchquery
        :param kwargs: kwargs to add to `q` as "key:value"
        """
        self._articles = []
        self.response = None  # current SolrResponse object
        self.max_pages = max_pages
        self.__iter_counter = 0  # Counter for our custom iterator method
        if query_dict is not None:
            query_dict.setdefault('rows', 50)
            query_dict.setdefault('start', 0)
            self._query = query_dict
        else:
            if sort is not None:
                sort = sort if " " in sort else "{} desc".format(sort)
            _ = {
                "q": q or '',
                "fq": fq,
                "fl": fl,
                "sort": sort,
                "start": start,
                "rows": int(rows),
            }
            # Filter out None values
            self._query = dict(
                (k, v) for k, v in six.iteritems(_) if v is not None
            )

            # Include `id` as a field, always (could be None, string or list)
            self._query.setdefault("fl", ["id"])
            if isinstance(self._query["fl"], six.string_types):
                _ = map(str.strip, self._query["fl"].split(","))
                self._query["fl"] = ["id"] + list(_)
            else:
                self._query["fl"] = ["id"] + self._query["fl"]

            # Format and add kwarg (key, value) pairs to q
            if kwargs:
                _ = ['{}:"{}"'.format(k, v) for k, v in six.iteritems(kwargs)]
                self._query['q'] = '{} {}'.format(self._query['q'], ' '.join(_))

        assert self._query.get('rows') > 0, "rows must be greater than 0"
        assert self._query.get('q'), "q must not be empty"

        if token is not None:
            self.token = token

def encode_task_value(value, query=False):
    if value is None:
        value = ''
    elif isinstance(value, datetime.datetime):
        if not value.tzinfo:
            #  Dates not having timezone information should be
            #  assumed to be in local time
            value = value.replace(tzinfo=dateutil.tz.tzlocal())
        #  All times should be converted to UTC before serializing
        value = value.astimezone(pytz.utc).strftime(DATE_FORMAT)
    elif isinstance(value, datetime.date):
        value = value.strftime(DATE_FORMAT)
    elif isinstance(value, six.string_types):
        if query:
            # In some contexts, parentheses are interpreted for use in
            # logical expressions.  They must *sometimes* be escaped.
            for left, right in six.iteritems(logical_replacements):
                value = value.replace(left, right)
        else:
            for unsafe, safe in six.iteritems(
                encode_replacements_experimental
            ):
                value = value.replace(unsafe, safe)
    else:
        value = str(value)
    return value

    def learn(self, my_dict, aplot=None):
        if my_dict is None:
            logging.critical("Cannot learn function with empty dict")
            return lambda _: 1, 0
        d_dict = dict()
        samples, thresholds = [], []
        for k, v in six.iteritems(my_dict):
            for o in (_ for _ in v if _):
                dnearest = np.array(np.load("{}.npz".format(o))['X']).reshape(
                    -1, 1)
                var = np.var(dnearest)
                if var == 0:
                    continue
                med = np.median(dnearest)
                mean, _, _, h = mean_confidence_interval(dnearest)
                samples.append(dnearest.shape[0])
                d_dict.setdefault(o.split('/')[0], dict()).setdefault(k, [med, h])

                # for the threshold, fit a gaussian (unused for AP)
                thresholds.append(_gaussian_fit(dnearest))
        if len(d_dict) < 1:
            logging.critical("dictionary is empty")
            return lambda _: 1, 0
        for k, v in six.iteritems(d_dict):  # there is only one
            xdata = np.array(sorted(v))
            ydata = np.array([np.mean(v[x][0]) for x in xdata])
            yerr = np.array([np.mean(v[x][1]) for x in xdata])

        # Take only significant values, higher than 0
        mask = ydata > 0
        xdata = xdata[mask]
        if xdata.shape[0] < 2:
            logging.critical("Too few points to learn function")
            # no correction can be applied
            return lambda _: 1, 0

        ydata = ydata[mask]
        ydata = ydata[0] / ydata  # normalise
        yerr = yerr[mask]

        order = min(self.order, xdata.shape[0] - 1)
        warnings.filterwarnings("ignore")
        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            try:
                poly = np.poly1d(np.polyfit(
                    xdata, ydata, order, w=1. / (yerr + 1e-15)))
            except np.RankWarning:
                logging.critical(
                    "Cannot fit polynomial with degree %d, npoints %d",
                    order, xdata.shape[0])
                return lambda _: 1, 0

        if self.aplot is not None:
            plot_learning_function(xdata, ydata, yerr, order, self.aplot, poly)

        # poly = partial(model, res.x)
        return poly, 1 - (filter(
            lambda x: x > 0,
            np.array(thresholds)[np.array(samples).argsort()[::-1]]) or [0])[0]

    def get_group_tag_values_for_users(self, event_users, limit=100):
        start, end = self.get_time_range()
        filters = {
            'project_id': [eu.project_id for eu in event_users]
        }
        conditions = [
            ['tags[sentry:user]', 'IN', filter(None, [eu.tag_value for eu in event_users])]
        ]
        aggregations = [
            ['count()', '', 'times_seen'],
            ['min', SEEN_COLUMN, 'first_seen'],
            ['max', SEEN_COLUMN, 'last_seen'],
        ]

        result = snuba.query(start, end, ['issue', 'user_id'], conditions, filters,
                             aggregations, orderby='-last_seen', limit=limit,
                             referrer='tagstore.get_group_tag_values_for_users')

        values = []
        for issue, users in six.iteritems(result):
            for name, data in six.iteritems(users):
                values.append(
                    GroupTagValue(
                        group_id=issue,
                        key='sentry:user',
                        value=name,
                        **fix_tag_value_data(data)
                    )
                )
        return values

    def _resolve_value(self, value):
        if isinstance(value, six.string_types):
            if value[0:2] == "<<":
                return value[1:]
            if value[0:1] == "<" and ">" in value:
                service_name, _, attrs = value[1:].rpartition(">")
                service = self._resolve_service(service_name)
                attrs = [a for a in attrs.split('.') if a]
                return six.moves.reduce(getattr, attrs, service)

        elif isinstance(value, dict):
            if self.factory_key in value:
                factory, args = self._resolve_value(value[self.factory_key]), []
                if isinstance(factory, list):
                    factory, args = factory[0], factory[1:]
                if isinstance(factory, six.string_types):
                    factory = import_name(factory)
                kwargs = {}
                for (dict_key, dict_value) in six.iteritems(value):
                    if dict_key != self.factory_key:
                        kwargs[dict_key] = self._resolve_value(dict_value)
                try:
                    return factory(*args, **kwargs)
                except Exception as err:
                    raise_and_annotate(err, "While calling factory '%s'" % value[self.factory_key])
            return {dict_key: self._resolve_value(dict_value)
                    for (dict_key, dict_value) in six.iteritems(value)}

        elif isinstance(value, list):
            return [self._resolve_value(item) for item in value]

        return value

    def outputconflicts(self, options):
        """saves the result of the conflict match"""
        print("%d/%d different strings have conflicts" % (len(self.conflictmap), len(self.textmap)))
        reducedmap = {}

        def str_len(x):
            return len(x)

        for source, translations in six.iteritems(self.conflictmap):
            words = source.split()
            words.sort(key=str_len)
            source = words[-1]
            reducedmap.setdefault(source, []).extend(translations)
        # reduce plurals
        plurals = {}
        for word in reducedmap:
            if word + "s" in reducedmap:
                plurals[word] = word + "s"
        for word, pluralword in six.iteritems(plurals):
            reducedmap[word].extend(reducedmap.pop(pluralword))
        for source, translations in six.iteritems(reducedmap):
            flatsource = self.flatten(source, "-")
            fulloutputpath = os.path.join(options.output, flatsource + os.extsep + "po")
            conflictfile = po.pofile()
            for target, unit, filename in translations:
                unit.othercomments.append("# (poconflicts) %s\n" % filename)
                conflictfile.units.append(unit)
            with open(fulloutputpath, "wb") as fh:
                conflictfile.serialize(fh)

def _write_f06_springs_transient(f, stress, header, words, name):
    for dt, datai in sorted(iteritems(data)):
        header[1] = ' %s = %10.4E\n' % (name, dt)
        msg += header + words
        f.write(''.join(msg))

        eids = []
        stresses = []
        for eid, stress in sorted(iteritems(datai)):
            eids.append(eid)
            stresses.append(stress)
            if len(stresses) == 4:
                stresses, is_all_zeros = writeFloats13E(stresses)
                f.write('    %10i  %13s    %10i  %13s    %10i  %13s    %10i  %13s\n' % (
                    eids[0], stresses[0],
                    eids[1], stresses[1],
                    eids[2], stresses[2],
                    eids[3], stresses[3]))
                eids = []
                stresses = []

        if stresses:
            line = '    '
            stresses, is_all_zeros = writeFloats13E(stresses)
            for eid, stress in zip(eids, stresses):
                line += '%10i  %13s    ' % (eid, stress)
            f.write(line.rstrip() + '\n')

            msg.append(page_stamp % page_num)
            f.write(''.join(msg))
            msg = ['']
        page_num += 1

    return page_num - 1

    def load_shabp_results(self, shabp_filename, dirname):
        Cpd, deltad = self.model.read_shabp_out(shabp_filename)

        cases = self.result_cases
        icase = len(cases)
        mach_results = []
        form = self.form
        form.append(('Results', None, mach_results))
        #self.result_cases = {}
        mach_forms = {}
        for case_id, Cp in sorted(iteritems(Cpd)):
            Cp = Cpd[case_id]
            #delta = deltad[case_id]

            mach, alpha, beta = self.model.shabp_cases[case_id]
            #name = 'Mach=%g Alpha=%g' % (mach, alpha)
            name = 'Mach=%g Alpha=%g' % (mach, alpha)
            cases[(name, icase, 'Cp', 1, 'centroid', '%.3f', '')] = Cp
            cp_form = [
                ('Cp', icase, [])
            ]
            mach_forms[mach].append(('Cp', None, cp_form))
            #self.result_cases[(name, 'delta', 1, 'centroid', '%.3f')] = delta

        for mach, mach_form in sorted(iteritems(mach_forms)):
            mach_results.append(mach_form)
        self._finish_results_io2(form, cases)

def parse_future_info(future_info):
    new_info = {}

    for underlying_symbol, info in six.iteritems(future_info):
        try:
            underlying_symbol = underlying_symbol.upper()
        except AttributeError:
            raise RuntimeError(_("Invalid future info: underlying_symbol {} is illegal.".format(underlying_symbol)))

        for field, value in six.iteritems(info):
            if field in (
                "open_commission_ratio", "close_commission_ratio", "close_commission_today_ratio"
            ):
                new_info.setdefault(underlying_symbol, {})[field] = float(value)
            elif field == "commission_type":
                if isinstance(value, six.string_types) and value.upper() == "BY_MONEY":
                    new_info.setdefault(underlying_symbol, {})[field] = COMMISSION_TYPE.BY_MONEY
                elif isinstance(value, six.string_types) and value.upper() == "BY_VOLUME":
                    new_info.setdefault(underlying_symbol, {})[field] = COMMISSION_TYPE.BY_VOLUME
                elif isinstance(value, COMMISSION_TYPE):
                    new_info.setdefault(underlying_symbol, {})[field] = value
                else:
                    raise RuntimeError(_(
                        "Invalid future info: commission_type is suppose to be BY_MONEY or BY_VOLUME"
                    ))
            else:
                raise RuntimeError(_("Invalid future info: field {} is not valid".format(field)))
    return new_info

def invert_dict(d, many = False):
    """
        Returns the inversion of keys and values in dictionary d. The default case
        allows overriding of values pointed to by multiple keys.  Passing many=True
        will make each value a key for the list of values pointing to it.

        Examples:
        invert_dict({ 1 : 2, 3 : 4 })
        { 2 : 1, 4 : 3 }

        invert_dict({ 1 : 2, 2 : 2, 3 : 2 })
        { 2 : 3 }

        invert_dict({ 1 : 2, 2 : 2, 3 : 2 }, many=True)
        { 2 : [ 1, 2, 3 ] }
    """
    if not many:
        return { v : k for k, v in six.iteritems(d) }

    output = collections.defaultdict(list)

    for k, v in six.iteritems(d):
        output[v].append(k)

    return dict(output)

    def allowed_permissions(self, package):
        """
        Get all allowed permissions for all principals on a package

        Returns
        -------
        perms : dict
            Mapping of principal to tuple of permissions

        """
        all_perms = {}
        for user, perms in six.iteritems(self.user_permissions(package)):
            all_perms["user:" + user] = tuple(perms)

        for group, perms in six.iteritems(self.group_permissions(package)):
            all_perms[group_to_principal(group)] = tuple(perms)

        # If there are no group or user specifications for the package, use the
        # default
        if not all_perms:
            for principal in groups_to_principals(self.default_read):
                all_perms[principal] = ("read",)
            for principal in groups_to_principals(self.default_write):
                if principal in all_perms:
                    all_perms[principal] += ("write",)
                else:
                    all_perms[principal] = ("write",)
        return all_perms

    def convex_cardinality_wrapper(*args, **kwargs):
        def dict_result(r):
            if isinstance(r, tuple):
                return dict(r[0])
            return dict(r)

        # Initial run with default weights
        full_result = f(*args, **kwargs)
        result = dict_result(full_result)

        def update_weight(value):
            return 1/(epsilon + abs(value))

        # Iterate until the difference from one iteration to
        # the next is less than epsilon.
        while True:
            weights = {identifier: update_weight(value)
                       for identifier, value in iteritems(result)}
            kwargs['weights'] = weights

            last_result = result
            full_result = f(*args, **kwargs)
            result = dict_result(full_result)

            delta = math.sqrt(sum(pow(value - last_result[identifier], 2)
                                  for identifier, value in iteritems(result)))
            if delta < epsilon:
                break

        if isinstance(full_result, tuple):
            return (iteritems(result),) + full_result[1:]
        return iteritems(result)