def test_footer_consensus_method_requirement(self):
    """
    Check that validation will notice if a footer appears before it was
    introduced.
    """

    content = get_network_status_document_v3({'consensus-method': '8'}, content = True)
    self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([DOC_SIG], document.signatures)
    self.assertEqual([], document.get_unrecognized_lines())

    # excludes a footer from a version that shouldn't have it

    document = get_network_status_document_v3({'consensus-method': '8'}, ('directory-footer', 'directory-signature'))
    self.assertEqual([], document.signatures)
    self.assertEqual([], document.get_unrecognized_lines())

    # Prior to conensus method 9 votes can still have a signature in their
    # footer...
    #
    # https://trac.torproject.org/7932

    document = get_network_status_document_v3(
      {
        'vote-status': 'vote',
        'consensus-methods': '1 8',
      },
      exclude = ('directory-footer',),
      authorities = (get_directory_authority(is_vote = True),)
    )

    self.assertEqual([DOC_SIG], document.signatures)
    self.assertEqual([], document.get_unrecognized_lines())

  def test_with_microdescriptor_router_status_entries(self):
    """
    Includes microdescriptor flavored router status entries within the
    document.
    """

    entry1 = get_router_status_entry_micro_v3({'s': "Fast"})
    entry2 = get_router_status_entry_micro_v3({'s': "Valid"})
    document = get_network_status_document_v3({"network-status-version": "3 microdesc"}, routers = (entry1, entry2))

    self.assertEquals((entry1, entry2), document.routers)

    # try with an invalid RouterStatusEntry

    entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False)
    content = get_network_status_document_v3({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True)

    self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEquals((entry3,), document.routers)

    # try including microdescriptor entries in a normal consensus

    content = get_network_status_document_v3(routers = (entry1, entry2), content = True)
    self.assertRaises(ValueError, NetworkStatusDocumentV3, content)

    expected_routers = (
      RouterStatusEntryV3(str(entry1), False),
      RouterStatusEntryV3(str(entry2), False),
    )

    document = NetworkStatusDocumentV3(content, False)
    self.assertEquals(expected_routers, document.routers)

  def test_with_microdescriptor_router_status_entries(self):
    """
    Includes microdescriptor flavored router status entries within the
    document.
    """

    entry1 = get_router_status_entry_micro_v3({'s': 'Fast'})
    entry2 = get_router_status_entry_micro_v3({
      'r': 'tornodeviennasil AcWxDFxrHetHYS5m6/MVt8ZN6AM 2013-03-13 22:09:13 78.142.142.246 443 80',
      's': 'Valid',
    })

    document = get_network_status_document_v3({'network-status-version': '3 microdesc'}, routers = (entry1, entry2))

    self.assertTrue(entry1 in document.routers.values())
    self.assertTrue(entry2 in document.routers.values())

    # try with an invalid RouterStatusEntry

    entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': 'ugabuga'}, content = True), False)

    content = get_network_status_document_v3({'network-status-version': '3 microdesc'}, routers = (entry3,), content = True)
    self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([entry3], list(document.routers.values()))

    # try including microdescriptor entry in a normal consensus

    content = get_network_status_document_v3(routers = (entry1,), content = True)
    self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([RouterStatusEntryV3(str(entry1), False)], list(document.routers.values()))

  def test_consensus_method(self):
    """
    Parses the consensus-method field.
    """

    document = get_network_status_document_v3({'consensus-method': '12'})
    self.assertEqual(12, document.consensus_method)

    # check that we default to being consensus-method 1
    content = get_network_status_document_v3(exclude = ('consensus-method',), content = True)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual(1, document.consensus_method)
    self.assertEqual([], document.consensus_methods)

    test_values = (
      '',
      '   ',
      'a',
      '1 2',
      '2.0',
    )

    for test_value in test_values:
      content = get_network_status_document_v3({'consensus-method': test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

      document = NetworkStatusDocumentV3(content, False)
      self.assertEqual(1, document.consensus_method)

  def test_consensus_method(self):
    """
    Parses the consensus-method field.
    """

    document = get_network_status_document_v3({"consensus-method": "12"})
    self.assertEquals(12, document.consensus_method)

    # check that we default to being consensus-method 1
    content = get_network_status_document_v3(exclude = ("consensus-method",), content = True)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEquals(1, document.consensus_method)
    self.assertEquals([], document.consensus_methods)

    test_values = (
      "",
      "   ",
      "a",
      "1 2",
      "2.0",
    )

    for test_value in test_values:
      content = get_network_status_document_v3({"consensus-method": test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content)

      document = NetworkStatusDocumentV3(content, False)
      self.assertEquals(1, document.consensus_method)

  def test_vote_status(self):
    """
    Parses the vote-status field.
    """

    document = get_network_status_document_v3({"vote-status": "vote"})
    self.assertEquals(False, document.is_consensus)
    self.assertEquals(True, document.is_vote)

    content = get_network_status_document_v3({"vote-status": "consensus"}, content = True)
    document = NetworkStatusDocumentV3(content)
    self.assertEquals(True, document.is_consensus)
    self.assertEquals(False, document.is_vote)

    test_values = (
      "",
      "   ",
      "votee",
    )

    for test_value in test_values:
      content = get_network_status_document_v3({"vote-status": test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content)

      document = NetworkStatusDocumentV3(content, False)
      self.assertEquals(True, document.is_consensus)
      self.assertEquals(False, document.is_vote)

  def test_with_router_status_entries(self):
    """
    Includes router status entries within the document. This isn't to test the
    RouterStatusEntry parsing but rather the inclusion of it within the
    document.
    """

    entry1 = get_router_status_entry_v3({'s': 'Fast'})
    entry2 = get_router_status_entry_v3({
      'r': 'Nightfae AWt0XNId/OU2xX5xs5hVtDc5Mes 6873oEfM7fFIbxYtwllw9GPDwkA 2013-02-20 11:12:27 85.177.66.233 9001 9030',
      's': 'Valid',
    })

    document = get_network_status_document_v3(routers = (entry1, entry2))

    self.assertTrue(entry1 in document.routers.values())
    self.assertTrue(entry2 in document.routers.values())

    # try with an invalid RouterStatusEntry

    entry3 = RouterStatusEntryV3(get_router_status_entry_v3({'r': 'ugabuga'}, content = True), False)
    content = get_network_status_document_v3(routers = (entry3,), content = True)

    self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([entry3], list(document.routers.values()))

    # try including with a microdescriptor consensus

    content = get_network_status_document_v3({'network-status-version': '3 microdesc'}, routers = (entry1,), content = True)
    self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([RouterStatusEntryMicroV3(str(entry1), False)], list(document.routers.values()))

  def test_version_lists(self):
    """
    Parses client-versions and server-versions fields. Both are comma separated
    lists of tor versions.
    """

    expected = [stem.version.Version("1.2.3.4"), stem.version.Version("56.789.12.34-alpha")]
    test_value = "1.2.3.4,56.789.12.34-alpha"

    document = get_network_status_document_v3({"client-versions": test_value, "server-versions": test_value})
    self.assertEquals(expected, document.client_versions)
    self.assertEquals(expected, document.server_versions)

    test_values = (
      ("", []),
      ("   ", []),
      ("1.2.3.4,", [stem.version.Version("1.2.3.4")]),
      ("1.2.3.4,1.2.3.a", [stem.version.Version("1.2.3.4")]),
    )

    for field in ('client-versions', 'server-versions'):
      attr = field.replace('-', '_')

      for test_value, expected_value in test_values:
        content = get_network_status_document_v3({field: test_value}, content = True)
        self.assertRaises(ValueError, NetworkStatusDocumentV3, content)

        document = NetworkStatusDocumentV3(content, False)
        self.assertEquals(expected_value, getattr(document, attr))

  def test_vote_status(self):
    """
    Parses the vote-status field.
    """

    document = get_network_status_document_v3({'vote-status': 'vote'})
    self.assertEqual(False, document.is_consensus)
    self.assertEqual(True, document.is_vote)

    content = get_network_status_document_v3({'vote-status': 'consensus'}, content = True)
    document = NetworkStatusDocumentV3(content)
    self.assertEqual(True, document.is_consensus)
    self.assertEqual(False, document.is_vote)

    test_values = (
      '',
      '   ',
      'votee',
    )

    for test_value in test_values:
      content = get_network_status_document_v3({'vote-status': test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

      document = NetworkStatusDocumentV3(content, False)
      self.assertEqual(True, document.is_consensus)
      self.assertEqual(False, document.is_vote)

  def test_consensus_methods(self):
    """
    Parses the consensus-methods field.
    """

    document = get_network_status_document_v3({'vote-status': 'vote', 'consensus-methods': '12 3 1 780'})
    self.assertEqual([12, 3, 1, 780], document.consensus_methods)

    # check that we default to including consensus-method 1
    content = get_network_status_document_v3({'vote-status': 'vote'}, ('consensus-methods',), content = True)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEqual([1], document.consensus_methods)
    self.assertEqual(None, document.consensus_method)

    test_values = (
      (''),
      ('   '),
      ('1 2 3 a 5'),
      ('1 2 3 4.0 5'),
      ('2 3 4'),  # spec says version one must be included
    )

    for test_value in test_values:
      content = get_network_status_document_v3({'vote-status': 'vote', 'consensus-methods': test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)

      expected_value = [2, 3, 4] if test_value == '2 3 4' else [1]

      document = NetworkStatusDocumentV3(content, False)
      self.assertEqual(expected_value, document.consensus_methods)

  def test_consensus_methods(self):
    """
    Parses the consensus-methods field.
    """

    document = get_network_status_document_v3({"vote-status": "vote", "consensus-methods": "12 3 1 780"})
    self.assertEquals([12, 3, 1, 780], document.consensus_methods)

    # check that we default to including consensus-method 1
    content = get_network_status_document_v3({"vote-status": "vote"}, ("consensus-methods",), content = True)
    document = NetworkStatusDocumentV3(content, False)
    self.assertEquals([1], document.consensus_methods)
    self.assertEquals(None, document.consensus_method)

    test_values = (
      ("", []),
      ("   ", []),
      ("1 2 3 a 5", [1, 2, 3, 5]),
      ("1 2 3 4.0 5", [1, 2, 3, 5]),
      ("2 3 4", [2, 3, 4]),  # spec says version one must be included
    )

    for test_value, expected_consensus_methods in test_values:
      content = get_network_status_document_v3({"vote-status": "vote", "consensus-methods": test_value}, content = True)
      self.assertRaises(ValueError, NetworkStatusDocumentV3, content)

      document = NetworkStatusDocumentV3(content, False)
      self.assertEquals(expected_consensus_methods, document.consensus_methods)

 def test_microdescriptor_signature(self):
   """
   The 'directory-signature' lines for normal and microdescriptor conensuses
   differ slightly in their format.
   """
   
   # including a microdescriptor flavored 'directory-signature' line should work
   
   document = get_network_status_document_v3({"network-status-version": "3 microdesc"})
   self.assertEqual('sha256', document.signatures[0].method)
   
   # include a standard 'directory-signature' line in a microdescriptor
   # consensus
   
   content = get_network_status_document_v3({
     "network-status-version": "3 microdesc",
     "directory-signature": NETWORK_STATUS_DOCUMENT_FOOTER[2][1],
   }, content = True)
   
   self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
   
   document = NetworkStatusDocumentV3(content, validate = False)
   self.assertEqual([], document.signatures)
   
   # includes a microdescriptor flavored 'directory-signature' line in a
   # normal consensus
   
   content = get_network_status_document_v3({
     "directory-signature": "sha256 " + NETWORK_STATUS_DOCUMENT_FOOTER[2][1],
   }, content = True)
   
   self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
   
   document = NetworkStatusDocumentV3(content, validate = False)
   self.assertEqual([], document.signatures)

  def test_duplicate_fields(self):
    """
    Almost all fields can only appear once. Checking that duplicates cause
    validation errors.
    """

    for is_consensus in (True, False):
      attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"}
      lines = get_network_status_document_v3(attr, content = True).split("\n")

      for index, line in enumerate(lines):
        # Stop when we hit the 'directory-signature' for a couple reasons...
        # - that is the one field that can validly appear multiple times
        # - after it is a crypto blob, which won't trigger this kind of
        #   validation failure

        test_lines = list(lines)
        if line.startswith("directory-signature "):
          break

        # duplicates the line
        test_lines.insert(index, line)

        content = "\n".join(test_lines)
        self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
        NetworkStatusDocumentV3(content, False)  # constructs without validation

  def test_unrecognized_line(self):
    """
    Includes unrecognized content in the document.
    """

    document = get_network_status_document_v3({"pepperjack": "is oh so tasty!"})
    self.assertEquals(["pepperjack is oh so tasty!"], document.get_unrecognized_lines())

  def test_examples(self):
    """
    Run something similar to the examples in the header pydocs.
    """

    # makes a consensus with a couple routers, both with the same nickname

    entry1 = get_router_status_entry_v3({'s': "Fast"})
    entry2 = get_router_status_entry_v3({'s': "Valid"})
    content = get_network_status_document_v3(routers = (entry1, entry2), content = True)

    # first example: parsing via the NetworkStatusDocumentV3 constructor

    consensus_file = StringIO.StringIO(content)
    consensus = NetworkStatusDocumentV3(consensus_file.read())
    consensus_file.close()

    for router in consensus.routers:
      self.assertEqual('caerSidi', router.nickname)

    # second example: using parse_file

    with support_with(StringIO.StringIO(content)) as consensus_file:
      for router in parse_file(consensus_file):
        self.assertEqual('caerSidi', router.nickname)

  def test_persisting_a_consensus(self, query_mock, open_mock, parse_file_mock, stdout_mock):
    def tutorial_example_1():
      from stem.descriptor import DocumentHandler
      from stem.descriptor.remote import DescriptorDownloader

      downloader = DescriptorDownloader()
      consensus = downloader.get_consensus(document_handler = DocumentHandler.DOCUMENT).run()[0]

      with open('/tmp/descriptor_dump', 'w') as descriptor_file:
        descriptor_file.write(str(consensus))

    def tutorial_example_2():
      from stem.descriptor import DocumentHandler, parse_file

      consensus = next(parse_file(
        '/tmp/descriptor_dump',
        descriptor_type = 'network-status-consensus-3 1.0',
        document_handler = DocumentHandler.DOCUMENT,
      ))

      for fingerprint, relay in consensus.routers.items():
        print('%s: %s' % (fingerprint, relay.nickname))

    network_status = get_network_status_document_v3(routers = (get_router_status_entry_v3(),))
    query_mock().run.return_value = [network_status]
    parse_file_mock.return_value = itertools.cycle([network_status])

    tutorial_example_1()
    tutorial_example_2()

    self.assertEqual(PERSISTING_A_CONSENSUS_OUTPUT, stdout_mock.getvalue())

  def test_examples(self):
    """
    Run something similar to the examples in the header pydocs.
    """

    # makes a consensus with a couple routers, both with the same nickname

    entry1 = get_router_status_entry_v3({'s': 'Fast'})
    entry2 = get_router_status_entry_v3({'s': 'Valid'})
    content = get_network_status_document_v3(routers = (entry1, entry2), content = True)

    # first example: parsing via the NetworkStatusDocumentV3 constructor

    consensus_file = io.BytesIO(content)
    consensus = NetworkStatusDocumentV3(consensus_file.read())
    consensus_file.close()

    for router in consensus.routers.values():
      self.assertEqual('caerSidi', router.nickname)

    # second example: using stem.descriptor.parse_file

    with io.BytesIO(content) as consensus_file:
      for router in stem.descriptor.parse_file(consensus_file, 'network-status-consensus-3 1.0'):
        self.assertEqual('caerSidi', router.nickname)

  def test_params_range(self):
    """
    Check both the furthest valid 'params' values and values that are out of
    bounds.
    """

    test_values = (
      ("foo=2147483648", {"foo": 2147483648}, False),
      ("foo=-2147483649", {"foo": -2147483649}, False),
      ("foo=2147483647", {"foo": 2147483647}, True),
      ("foo=-2147483648", {"foo": -2147483648}, True),

      # param with special range constraints
      ("circwindow=99", {"circwindow": 99}, False),
      ("circwindow=1001", {"circwindow": 1001}, False),
      ("circwindow=500", {"circwindow": 500}, True),

      # param that relies on another param for its constraints
      ("cbtclosequantile=79 cbtquantile=80", {"cbtclosequantile": 79, "cbtquantile": 80}, False),
      ("cbtclosequantile=80 cbtquantile=80", {"cbtclosequantile": 80, "cbtquantile": 80}, True),
    )

    for test_value, expected_value, is_ok in test_values:
      content = get_network_status_document_v3({"params": test_value}, content = True)

      if is_ok:
        document = NetworkStatusDocumentV3(content, default_params = False)
      else:
        self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
        document = NetworkStatusDocumentV3(content, False, default_params = False)

      self.assertEquals(expected_value, document.params)

  def test_misordered_fields(self):
    """
    Rearranges our descriptor fields.
    """

    for is_consensus in (True, False):
      attr = {'vote-status': 'consensus'} if is_consensus else {'vote-status': 'vote'}
      lines = get_network_status_document_v3(attr, content = True).split(b'\n')

      for index in range(len(lines) - 1):
        # once we reach the authority entry or later we're done since swapping
        # those won't be detected

        if is_consensus and lines[index].startswith(stem.util.str_tools._to_bytes(CRYPTO_BLOB[1:10])):
          break
        elif not is_consensus and lines[index].startswith(b'dir-source'):
          break

        # swaps this line with the one after it
        test_lines = list(lines)
        test_lines[index], test_lines[index + 1] = test_lines[index + 1], test_lines[index]

        content = b'\n'.join(test_lines)
        self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True)
        NetworkStatusDocumentV3(content, False)  # constructs without validation

  def test_unrecognized_line(self):
    """
    Includes unrecognized content in the document.
    """

    document = get_network_status_document_v3({'pepperjack': 'is oh so tasty!'})
    self.assertEqual(['pepperjack is oh so tasty!'], document.get_unrecognized_lines())