def test_look_for_defs(self, node_definitions):
        """We should be walking through the tree to find terms. Test this by
        documenting which nodes are touched. We should be _ignoring_ certain
        subtrees (notable, any which aren't associated w/ regtext)"""
        node_definitions.side_effect = lambda n, _: ([], [n.label_id()])
        t = Terms(None)

        root = Node(label=['111'], children=[
            Node(label=['111', 'Subpart'], node_type=Node.EMPTYPART, children=[
                Node(label=['111', '1'], children=[
                    Node(label=['111', '1', 'a']),
                    Node(label=['111', '1', 'b']),
                    Node(label=['111', '1', 'c'])]),
                Node(label=['111', '2'], children=[
                    Node(label=['111', '2', 'p1'], node_type=Node.EXTRACT,
                         children=[Node(label=['111', '2', 'p1', 'p1'])])
                ])]),
            Node(label=['111', 'A'], node_type=Node.APPENDIX, children=[
                Node(label=['111', 'A', '1'], node_type=Node.APPENDIX)])])
        t.look_for_defs(root)
        six.assertCountEqual(
            self,
            t.scoped_terms['EXCLUDED'],
            # note the absence of APPENDIX, and anything below an EXTRACT
            ['111', '111-Subpart', '111-1', '111-1-a', '111-1-b', '111-1-c',
             '111-2'])

 def test_process(self):
     """The process() method should both find terms in the requested node
     and order them by term name"""
     t = Terms(Node(children=[
         Node("ABC5", children=[Node("child")], label=['ref1']),
         Node("AABBCC5", label=['ref2']),
         Node("ABC3", label=['ref3']),
         Node("AAA3", label=['ref4']),
         Node("ABCABC3", label=['ref5']),
         Node("ABCOTHER", label=['ref6']),
         Node("ZZZOTHER", label=['ref7']),
     ]))
     t.scoped_terms = {
         ("101", "22", "b", "2", "ii"): [
             Ref("abc", "ref1", 1),
             Ref("aabbcc", "ref2", 2)],
         ("101", "22", "b"): [
             Ref("abc", "ref3", 3),
             Ref("aaa", "ref4", 4),
             Ref("abcabc", "ref5", 5)],
         ("101", "22", "b", "2", "iii"): [
             Ref("abc", "ref6", 6),
             Ref("zzz", "ref7", 7)]}
     #   Check that the return value is correct
     layer_el = t.process(Node(
         "This has abc, aabbcc, aaa, abcabc, and zzz",
         label=["101", "22", "b", "2", "ii"]))
     self.assertEqual(
         [el['ref'] for el in layer_el],
         ['aaa:ref4', 'aabbcc:ref2', 'abc:ref1', 'abcabc:ref5'])

 def test_node_definitions(self):
     t = Terms(None)
     text1 = u'This has a “worD” and then more'
     text2 = u'I have “anotheR word” term and “moree”'
     text3 = u'This has no defs'
     text3a = u'But the child “DoeS sEe”?'
     text3bi = u'As do “subchildren”'
     text3biA = u'Also has no terms'
     text3bii = u'Has no terms'
     text3c = u'Also has no terms'
     tree = Node(children=[ 
         Node(text1, label=['aaa']),
         Node(text2, label=['bbb']),
         Node(text3, children=[ 
             Node(text3a, label=['ccc']),
             Node(children=[ 
                 Node(text3bi, [Node(text3biA)], ['ddd']),
                 Node(text3bii)
             ]),
             Node(text3c)
         ])
     ])
     defs, excluded = t.node_definitions(tree)
     self.assertEqual(5, len(defs))
     self.assertTrue(Ref('word', 'aaa', (12,16)) in defs)
     self.assertTrue(Ref('another word', 'bbb', (8,20)) in defs)
     self.assertTrue(Ref('moree', 'bbb', (32,37)) in defs)
     self.assertTrue(Ref('does see', 'ccc', (15,23)) in defs)
     self.assertTrue(Ref('subchildren', 'ddd', (7,18)) in defs)

 def test_excluded_offsets_blacklist(self):
     t = Terms(None)
     t.scoped_terms['_'] = [Ref('bourgeois', '12-Q-2', 0)]
     settings.IGNORE_DEFINITIONS_IN['ALL'] = ['bourgeois pig']
     excluded = t.excluded_offsets(Node('You are a bourgeois pig!',
                                        label=['12', '3']))
     self.assertEqual([(10, 23)], excluded)

 def test_calculate_offsets_exclusions(self):
     applicable_terms = [('act', 'a')]
     text = "This text defines the 'fudge act'"
     t = Terms(None)
     self.assertEqual([], 
             t.calculate_offsets(text, applicable_terms, [(23,32)]))
     self.assertEqual([('act', 'a', [(29,32)])],
             t.calculate_offsets(text, applicable_terms, [(1,5)]))

 def test_calculate_offsets_word_part(self):
     """If a defined term is part of another word, don't include it"""
     applicable_terms = [('act', 'a')]
     text = "I am about to act on this transaction."
     t = Terms(None)
     matches = t.calculate_offsets(text, applicable_terms)
     self.assertEqual(1, len(matches))
     self.assertEqual(1, len(matches[0][2]))

 def test_calculate_offsets_overlap(self):
     applicable_terms = [('mad cow disease', 'mc'), ('goes mad', 'gm')]
     text = 'There goes mad cow disease'
     t = Terms(None)
     matches = t.calculate_offsets(text, applicable_terms)
     self.assertEqual(1, len(matches))
     _, ref, offsets = matches[0]
     self.assertEqual('mc', ref)
     self.assertEqual('mad cow disease', text[offsets[0][0]:offsets[0][1]])

 def test_calculate_offsets_lexical_container(self):
     applicable_terms = [('access device', 'a'), ('device', 'd')]
     text = "This access device is fantastic!"
     t = Terms(None)
     matches = t.calculate_offsets(text, applicable_terms)
     self.assertEqual(1, len(matches))
     _, ref, offsets = matches[0]
     self.assertEqual('a', ref)
     self.assertEqual([(5, 18)], offsets)

 def test_node_definitions_needs_term(self):
     t = Terms(None)
     stack = ParentStack()
     stack.add(0, Node('Definitions', label=['9999']))
     node = Node(u"However, for purposes of rescission under §§ 1111.15 "
                 + u"and 1111.13, and for purposes of §§ 1111.12(a)(1), "
                 + u"and 1111.46(d)(4), the term means all calendar "
                 + u"days...")
     self.assertEqual(([], []), t.node_definitions(node, stack))

    def test_pre_process_defined_twice(self):
        tree = Node(u"The term “lol” means laugh out loud. "
                    + u"How do you pronounce “lol”, though?",
                    label=['1212', '5'])
        t = Terms(tree)
        t.pre_process()

        self.assertEqual(t.layer['referenced']['lol:1212-5']['position'],
                         (10, 13))

    def test_node_defintions_act(self):
        t = Terms(None)
        stack = ParentStack()
        stack.add(0, Node('Definitions', label=['9999']))

        node = Node(u'“Act” means something else entirely')
        included, excluded = t.node_definitions(node, stack)
        self.assertEqual(1, len(included))
        self.assertEqual([], excluded)

    def test_pre_process(self):
        noname_subpart = Node('', label=['88', 'Subpart'],
            node_type=Node.EMPTYPART, children=[
                Node(u"Definition. For the purposes of this part, "
                     + u"“abcd” is an alphabet", label=['88', '1'])])
        xqxq_subpart = Node('', title='Subpart XQXQ: The unreadable',
            label=['88', 'Subpart', 'XQXQ'], node_type=Node.SUBPART,
            children=[
                Node(label=['88', '2'], children=[
                    Node(label=['88', '2', 'a'],
                         text="Definitions come later for the purposes of "
                              + "this section ", children=[
                            Node(u"“AXAX” means axe-cop",
                                 label=['88', '2', 'a', '1'])]),
                    Node(label=['88', '2', 'b'], children=[
                        Node(label=['88', '2', 'b', 'i'], children=[
                            Node(label=['88', '2', 'b', 'i', 'A'],
                                 text=u"Definition. “Awesome sauce” means "
                                      +"great for the purposes of this "
                                      + "paragraph",)])])])])
        tree = Node(label=['88'], children=[noname_subpart, xqxq_subpart])
        t = Terms(tree)
        t.pre_process()

        self.assertTrue(('88',) in t.scoped_terms)
        self.assertEqual([Ref('abcd', '88-1', (44,48))], 
                t.scoped_terms[('88',)])
        self.assertTrue(('88','2') in t.scoped_terms)
        self.assertEqual([Ref('axax', '88-2-a-1', (1,5))], 
            t.scoped_terms[('88','2')])
        self.assertTrue(('88','2','b','i','A') in t.scoped_terms)
        self.assertEqual([Ref('awesome sauce', '88-2-b-i-A', (13,26))], 
                t.scoped_terms[('88','2','b','i','A')])

        #   Check subparts are correct
        self.assertEqual({None: ['1'], 'XQXQ': ['2']}, dict(t.subpart_map))

        # Finally, make sure the references are added
        referenced = t.layer['referenced']
        self.assertTrue('abcd:88-1' in referenced)
        self.assertEqual('abcd', referenced['abcd:88-1']['term'])
        self.assertEqual('88-1', referenced['abcd:88-1']['reference'])
        self.assertEqual((44,48), referenced['abcd:88-1']['position'])

        self.assertTrue('axax:88-2-a-1' in referenced)
        self.assertEqual('axax', referenced['axax:88-2-a-1']['term'])
        self.assertEqual('88-2-a-1', referenced['axax:88-2-a-1']['reference'])
        self.assertEqual((1,5), referenced['axax:88-2-a-1']['position'])

        self.assertTrue('awesome sauce:88-2-b-i-A' in referenced)
        self.assertEqual('awesome sauce', 
            referenced['awesome sauce:88-2-b-i-A']['term'])
        self.assertEqual('88-2-b-i-A', 
            referenced['awesome sauce:88-2-b-i-A']['reference'])
        self.assertEqual((13,26), 
            referenced['awesome sauce:88-2-b-i-A']['position'])

 def test_calculate_offsets(self):
     applicable_terms = [('rock band', 'a'), ('band', 'b'), ('drum', 'c'),
                         ('other thing', 'd')]
     text = "I am in a rock band. That's a band with a drum, a rock drum."
     t = Terms(None)
     matches = t.calculate_offsets(text, applicable_terms)
     self.assertItemsEqual(matches, [
         ('rock band', 'a', [(10, 19)]),
         ('band', 'b', [(30, 34)]),
         ('drum', 'c', [(42, 46), (55, 59)])])

 def test_excluded_offsets(self):
     t = Terms(None)
     t.scoped_terms['_'] = [
         Ref('term', 'lablab', (4, 6)), Ref('other', 'lablab', (8, 9)),
         Ref('more', 'nonnon', (1, 8))
     ]
     self.assertEqual([(4, 6), (8, 9)],
                      t.excluded_offsets('lablab', 'Some text'))
     self.assertEqual([(1, 8)], t.excluded_offsets('nonnon', 'Other'))
     self.assertEqual([], t.excluded_offsets('ababab', 'Ab ab ab'))

 def test_node_definitions_exclusion(self):
     t = Terms(None)
     node = Node('',[
         Node(u'“Bologna” is a type of deli meat',
             label=['1']),
         Node(u'Let us not forget that the term “bologna” does not ' +
             'include turtle meat', label=['2'])
     ])
     included, excluded = t.node_definitions(node)
     self.assertEqual([Ref('bologna', '1', (1,8))], included)
     self.assertEqual([Ref('bologna', '2', (33,40))], excluded)

    def test_node_defintions_act(self):
        t = Terms(None)
        node = Node(u'“Act” means some reference to 99 U.S.C. 1234')
        included, excluded = t.node_definitions(node)
        self.assertEqual([], included)
        self.assertEqual(1, len(excluded))

        node = Node(u'“Act” means something else entirely')
        included, excluded = t.node_definitions(node)
        self.assertEqual(1, len(included))
        self.assertEqual([], excluded)

 def test_excluded_offsets_blacklist_word_boundaries(self):
     """If an exclusion begins/ends with word characters, the searching
     regex should make sure to only match on word boundaries"""
     settings.IGNORE_DEFINITIONS_IN['ALL'] = ['shed act', '(phrase)']
     t = Terms(None)
     t.scoped_terms['_'] = [Ref('act', '28-6-d', 0)]
     excluded = t.excluded_offsets(Node("That's a watershed act",
                                        label=['28', '9']))
     self.assertEqual([], excluded)
     excluded = t.excluded_offsets(Node("This has a '(phrase)' in it",
                                        label=['28', '9']))
     self.assertNotEqual([], excluded)

    def test_excluded_offsets_blacklist_per_reg(self):
        t = Terms(None)

        t.scoped_terms['_'] = [
            Ref('bourgeois', '12-Q-2', 0),
            Ref('consumer', '12-Q-3', 0)]

        settings.IGNORE_DEFINITIONS_IN['ALL'] = ['bourgeois pig']
        settings.IGNORE_DEFINITIONS_IN['12'] = ['consumer price index']
        excluded = t.excluded_offsets(
            Node('There is a consumer price index', label=['12', '2']))
        self.assertEqual([(11, 31)], excluded)

    def test_determine_scope(self):
        stack = ParentStack()
        t = Terms(None)

        stack.add(0, Node(label=['1000']))
        stack.add(1, Node(label=['1000', '1']))

        # Defaults to the entire reg
        self.assertEqual([('1000',)], t.determine_scope(stack))

        stack.add(1, Node('For the purposes of this part, blah blah',
                          label=['1001', '2']))
        self.assertEqual([('1001',), ('1001', Node.INTERP_MARK)],
                         t.determine_scope(stack))

        t.subpart_map = {
            'SubPart 1': ['A', '3'],
            'Other': []
        }
        stack.add(1, Node(label=['1000', '3']))
        stack.add(2, Node('For the purposes of this subpart, yada yada',
                          label=['1000', '3', 'c']))
        self.assertEqual([('1000', 'A'), ('1000', '3'),
                          ('1000', 'A', Node.INTERP_MARK),
                          ('1000', '3', Node.INTERP_MARK)],
                         t.determine_scope(stack))

        stack.add(2, Node('For the purposes of this section, blah blah',
                          label=['1000', '3', 'd']))
        self.assertEqual([('1000', '3'), ('1000', '3', Node.INTERP_MARK)],
                         t.determine_scope(stack))

        stack.add(3, Node('For the purposes of this paragraph, blah blah',
                          label=['1000', '3', 'd', '5']))
        self.assertEqual([('1000', '3', 'd', '5'),
                          ('1000', '3', 'd', '5', Node.INTERP_MARK)],
                         t.determine_scope(stack))

        stack.add(3, Node(label=['1002', '3', 'd', '6']))
        self.assertEqual([('1000', '3'), ('1000', '3', Node.INTERP_MARK)],
                         t.determine_scope(stack))
        stack.add(4, Node(u'For the purposes of this § 1000.3(d)(6)(i), blah',
                          label=['1000', '3', 'd', '6', 'i']))
        self.assertEqual([('1000', '3', 'd', '6', 'i'),
                          ('1000', '3', 'd', '6', 'i', Node.INTERP_MARK)],
                         t.determine_scope(stack))

        stack.add(4, Node(u'For the purposes of § 1000.3, blah',
                          label=['1000', '3', 'd', '6', 'ii']))
        self.assertEqual([('1000', '3'),
                          ('1000', '3', Node.INTERP_MARK)],
                         t.determine_scope(stack))

    def test_pre_process(self):
        settings.SUBPART_STARTS = {'2': 'XQXQ'}
        tree = Node(children=[
            Node(u"Definition. For the purposes of this part, “abcd” is "
                + "an alphabet", label=['88', '1']),
            Node(children=[
                Node("Definitions come later for the purposes of this "
                    + "section ", children=[
                        Node(u"“AXAX” means axe-cop", label=["88","2","a","1"])
                    ],
                    label=["88","2","a"]),
                Node(children=[Node(children=[Node(
                    u"Definition. “Awesome sauce” means great for the " +
                    "purposes of this paragraph", 
                    label=["88","2","b","i","A"])])])
            ], label=['88','2'])
        ])
        t = Terms(tree)
        t.pre_process()

        self.assertTrue(('88',) in t.scoped_terms)
        self.assertEqual([Ref('abcd', '88-1', (44,48))], 
                t.scoped_terms[('88',)])
        self.assertTrue(('88','2') in t.scoped_terms)
        self.assertEqual([Ref('axax', '88-2-a-1', (1,5))], 
            t.scoped_terms[('88','2')])
        self.assertTrue(('88','2','b','i','A') in t.scoped_terms)
        self.assertEqual([Ref('awesome sauce', '88-2-b-i-A', (13,26))], 
                t.scoped_terms[('88','2','b','i','A')])

        #   Check subparts are correct
        self.assertEqual({None: ['1'], 'XQXQ': ['2']}, dict(t.subpart_map))

        # Finally, make sure the references are added
        referenced = t.layer['referenced']
        self.assertTrue('abcd:88-1' in referenced)
        self.assertEqual('abcd', referenced['abcd:88-1']['term'])
        self.assertEqual('88-1', referenced['abcd:88-1']['reference'])
        self.assertEqual((44,48), referenced['abcd:88-1']['position'])

        self.assertTrue('axax:88-2-a-1' in referenced)
        self.assertEqual('axax', referenced['axax:88-2-a-1']['term'])
        self.assertEqual('88-2-a-1', referenced['axax:88-2-a-1']['reference'])
        self.assertEqual((1,5), referenced['axax:88-2-a-1']['position'])

        self.assertTrue('awesome sauce:88-2-b-i-A' in referenced)
        self.assertEqual('awesome sauce', 
            referenced['awesome sauce:88-2-b-i-A']['term'])
        self.assertEqual('88-2-b-i-A', 
            referenced['awesome sauce:88-2-b-i-A']['reference'])
        self.assertEqual((13,26), 
            referenced['awesome sauce:88-2-b-i-A']['position'])