def _from_word2vec_text(fname):
   with _open(fname, 'rb') as fin:
     words = []
     header = unicode(fin.readline())
     vocab_size, layer1_size = list(map(int, header.split())) # throws for invalid file format
     vectors = []
     for line_no, line in enumerate(fin):
       try:
         parts = unicode(line, encoding="utf-8").strip().split()
       except TypeError as e:
         parts = line.strip().split()
       except Exception as e:
         logger.warning("We ignored line number {} because of erros in parsing"
                         "\n{}".format(line_no, e))
         continue
       # We differ from Gensim implementation.
       # Our assumption that a difference of one happens because of having a
       # space in the word.
       if len(parts) == layer1_size + 1:
         word, weights = parts[0], list(map(float32, parts[1:]))
       elif len(parts) == layer1_size + 2:
         word, weights = parts[:2], list(map(float32, parts[2:]))
         word = u" ".join(word)
       else:
         logger.warning("We ignored line number {} because of unrecognized "
                         "number of columns {}".format(line_no, parts[:-layer1_size]))
         continue
       index = line_no
       words.append(word)
       vectors.append(weights)
     vectors = np.asarray(vectors, dtype=np.float32)
     return words, vectors

 def unknown_starttag(self, tag, attrs):
     # called for each start tag
     # attrs is a list of (attr, value) tuples
     # e.g. for <pre class='screen'>, tag='pre', attrs=[('class', 'screen')]
     uattrs = []
     strattrs = ''
     if attrs:
         for key, value in attrs:
             value = value.replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;')
             value = self.bare_ampersand.sub("&amp;", value)
             # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds
             if isinstance(value, unicode):
                 try:
                     value = unicode(value, self.encoding)
                 except:
                     value = unicode(value, 'iso-8859-1')
             uattrs.append((unicode(key, self.encoding), value))
         strattrs = u''.join([u' %s="%s"' % (key, val) for key, val in uattrs])
         if self.encoding:
             try:
                 strattrs = strattrs.encode(self.encoding)
             except:
                 pass
     if tag in self.elements_no_end_tag:
         self.pieces.append('<%(tag)s%(strattrs)s />' % locals())
     else:
         self.pieces.append('<%(tag)s%(strattrs)s>' % locals())

def stream_from_fh(fh, clean=False):
    for l in fh:
        l = l.decode('utf-8')
        try:
            yield Word.from_string(unicode(l), clean=clean)
        except ValueError as v:
            print(unicode(v).encode('utf-8'))
            continue

 def _get_message(self, run_errors, teardown_errors):
     run_msg = unicode(run_errors or '')
     td_msg = unicode(teardown_errors or '')
     if not td_msg:
         return run_msg
     if not run_msg:
         return 'Keyword teardown failed:\n%s' % td_msg
     return '%s\n\nAlso keyword teardown failed:\n%s' % (run_msg, td_msg)

 def _parse_arguments(self, cli_args):
     try:
         options, arguments = self.parse_arguments(cli_args)
     except Information as msg:
         self._report_info(unicode(msg))
     except DataError as err:
         self._report_error(unicode(err), help=True, exit=True)
     else:
         self._logger.info('Arguments: %s' % ','.join(arguments))
         return options, arguments

    def dictionary_should_contain_item(self, dictionary, key, value, msg=None):
        """An item of `key`/`value` must be found in a `dictionary`.

        Value is converted to unicode for comparison.

        See `Lists Should Be Equal` for an explanation of `msg`.
        The given dictionary is never altered by this keyword.
        """
        self.dictionary_should_contain_key(dictionary, key, msg)
        actual, expected = unicode(dictionary[key]), unicode(value)
        default = "Value of dictionary key '%s' does not match: %s != %s" % (key, actual, expected)
        _verify_condition(actual == expected, default, msg)

    def copy(cls, url_obj):
        if not isinstance(url_obj, Url):
            raise TypeError('url_obj should be an Url object')

        u = Url(unicode(url_obj.entity_name))
        entity_id = url_obj.entity_id
        u.entity_id = unicode(entity_id) if entity_id is not None else None
        u.path = list(url_obj.path)
        u.prefix = unicode(url_obj.prefix)
        u._query = dict(url_obj._query)

        return u

 def test_run_revsort(self):
     outDir = self._createTempDir()
     self._tester('src/toil/test/cwl/revsort.cwl',
                  'src/toil/test/cwl/revsort-job.json',
                  outDir, {
         # Having unicode string literals isn't necessary for the assertion but makes for a
         # less noisy diff in case the assertion fails.
         u'output': {
             u'path': unicode(os.path.join(outDir, 'output.txt')),
             u'basename': unicode("output.txt"),
             u'size': 1111,
             u'class': u'File',
             u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

 def _populate_children(self, datadir, children, include_suites, warn_on_skipped):
     for child in children:
         try:
             datadir.add_child(child, include_suites)
         except DataError as err:
             self._log_failed_parsing("Parsing data source '%s' failed: %s"
                         % (child, unicode(err)), warn_on_skipped)

def add(message, history_dir):
    _check_history_dir(history_dir)
    message = unicode(message)
    hashed = _make_hash_name(message)
    filepath = history_dir / hashed
    with filepath.open('w') as history_entry:
        history_entry.write(message + '\n')

 def test_import_non_existing_module(self):
     msg = ("Importing test library '%s' failed: ImportError: No module named "
            + ("'%s'" if PY3 else "%s"))
     for name in 'nonexisting', 'nonexi.sting':
         error = assert_raises(DataError, TestLibrary, name)
         assert_equals(unicode(error).splitlines()[0],
                       msg % (name, name.split('.')[0]))

 def _parse(self, path):
     try:
         return TestData(source=abspath(path),
                         include_suites=self.include_suites,
                         warn_on_skipped=self.warn_on_skipped)
     except DataError as err:
         raise DataError("Parsing '%s' failed: %s" % (path, unicode(err)))

def annotate_links(answer_text):
    """
    Parse and annotate links from answer text and return the annotated answer
    and an enumerated list of links as footnotes.
    """

    try:
        SITE = Site.objects.get(is_default_site=True)
    except Site.DoesNotExist:
        raise RuntimeError('no default wagtail site configured')

    footnotes = []
    soup = bs(answer_text, 'lxml')
    links = soup.findAll('a')
    index = 1
    for link in links:
        if not link.get('href'):
            continue
        footnotes.append(
            (index, urljoin(SITE.root_url, link.get('href'))))
        parent = link.parent
        link_location = parent.index(link)
        super_tag = soup.new_tag('sup')
        super_tag.string = str(index)
        parent.insert(link_location + 1, super_tag)
        index += 1
    return (unicode(soup), footnotes)

 def start_keyword(self, kw):
     attrs = {'name': kw.name, 'type': kw.type}
     if kw.timeout:
         attrs['timeout'] = unicode(kw.timeout)
     self._writer.start('kw', attrs)
     self._writer.element('doc', kw.doc)
     self._write_list('arguments', 'arg', (unic(a) for a in kw.args))

def _build_series(series, dim_names, comment, delta_name, delta_unit):
    from glue.ligolw import array as ligolw_array
    Attributes = ligolw.sax.xmlreader.AttributesImpl
    elem = ligolw.LIGO_LW(
            Attributes({u"Name": unicode(series.__class__.__name__)}))
    if comment is not None:
        elem.appendChild(ligolw.Comment()).pcdata = comment
    elem.appendChild(ligolw.Time.from_gps(series.epoch, u"epoch"))
    elem.appendChild(ligolw_param.Param.from_pyvalue(u"f0", series.f0,
                                                     unit=u"s^-1"))
    delta = getattr(series, delta_name)
    if numpy.iscomplexobj(series.data.data):
        data = numpy.row_stack((numpy.arange(len(series.data.data)) * delta,
                             series.data.data.real, series.data.data.imag))
    else:
        data = numpy.row_stack((numpy.arange(len(series.data.data)) * delta,
                                series.data.data))
    a = ligolw_array.Array.build(series.name, data, dim_names=dim_names)
    a.Unit = str(series.sampleUnits)
    dim0 = a.getElementsByTagName(ligolw.Dim.tagName)[0]
    dim0.Unit = delta_unit
    dim0.Start = series.f0
    dim0.Scale = delta
    elem.appendChild(a)
    return elem

 def test_structure(self):
     error = self._failing_import('NoneExisting')
     message = ("Importing 'NoneExisting' failed: ImportError: No module named "
                + ("'%s'" if PY3 else "%s") % 'NoneExisting')
     expected = (message, self._get_traceback(error),
                 self._get_pythonpath(error), self._get_classpath(error))
     assert_equals(unicode(error), '\n'.join(expected).strip())

    def child(self, logger, action_type, serializers=None):
        """
        Create a child L{Action}.

        Rather than calling this directly, you can use L{startAction} to
        create child L{Action} using the execution context.

        @param logger: The L{eliot.ILogger} to which to write
            messages.

        @param action_type: The type of this action,
            e.g. C{"yourapp:subsystem:dosomething"}.

        @param serializers: Either a L{eliot._validation._ActionSerializers}
            instance or C{None}. In the latter case no validation or
            serialization will be done for messages generated by the
            L{Action}.
        """
        self._numberOfChildren += 1
        newLevel = (self._identification["task_level"] +
                    unicode(self._numberOfChildren) + "/")
        return self.__class__(logger,
                              self._identification["task_uuid"],
                              newLevel,
                              action_type,
                              serializers)

def last_journald_message():
    """
    @return: Last journald message from this process as a dictionary in
         journald JSON format.
    """
    # It may take a little for messages to actually reach journald, so we
    # write out marker message and wait until it arrives. We can then be
    # sure the message right before it is the one we want.
    marker = unicode(uuid4())
    sd_journal_send(MESSAGE=marker.encode("ascii"))
    for i in range(500):
        messages = check_output(
            [
                b"journalctl",
                b"-a",
                b"-o",
                b"json",
                b"-n2",
                b"_PID=" + str(getpid()).encode("ascii"),
            ]
        )
        messages = [loads(m) for m in messages.splitlines()]
        if len(messages) == 2 and messages[1]["MESSAGE"] == marker:
            return messages[0]
        sleep(0.01)
    raise RuntimeError("Message never arrived?!")

 def from_word2vec_vocab(fvocab):
   counts = {}
   with _open(fvocab) as fin:
     for line in fin:
       word, count = unicode(line).strip().split()
       counts[word] = int(count)
   return CountedVocabulary(word_count=counts)

    def _freeze(self, action=None):
        """
        Freeze this message for logging, registering it with C{action}.

        @param action: The L{Action} which is the context for this message. If
            C{None}, the L{Action} will be deduced from the current call
            stack.

        @return: A L{PMap} with added C{timestamp}, C{task_uuid}, and
            C{task_level} entries.
        """
        if action is None:
            action = current_action()
        if action is None:
            task_uuid = unicode(uuid4())
            task_level = [1]
        else:
            task_uuid = action._identification[TASK_UUID_FIELD]
            task_level = action._nextTaskLevel().as_list()
        timestamp = self._timestamp()
        new_values = {
            TIMESTAMP_FIELD: timestamp,
            TASK_UUID_FIELD: task_uuid,
            TASK_LEVEL_FIELD: task_level,
        }
        if "action_type" not in self._contents and (
            "message_type" not in self._contents
        ):
            new_values["message_type"] = ""
        new_values.update(self._contents)
        return new_values