def scan_field_id(token, state, stream, lexdata):
    # When a candidate is found, do as follows:
    # - save the candidate
    # - eat any whitespace
    # - if next is colon, candidate is an identifier, emit both
    # identifier and colon
    candidate = token
    token = stream.peek()
    if token.type == "WS":
        token = stream.peek()
    if token.type == "COLON":
        # We do have a identifier, so replace WORD token by the
        # right keyword token
        candidate = _new_token(META_FIELDS_ID[candidate.value], candidate)

    try:
        field_type = FIELD_TYPE[candidate.type]
    except KeyError:
        data = lexdata.splitlines()
        msg = ["Error while tokenizing %r (missing colon ?)" %  candidate.value]
        msg += ["    Line %d -> %r" % (candidate.lineno, data[candidate.lineno-1])]
        raise SyntaxError("\n".join(msg))
    try:
        state = _FIELD_TYPE_TO_STATE[field_type]
    except KeyError:
        raise ValueError("Unknown state transition for type %s" % field_type)

    queue = [candidate]
    queue.append(six.advance_iterator(stream))
    nxt = six.advance_iterator(stream)
    return queue, nxt, state

 def __delitem__(self, key):
     tmp = self._slice.duplicate()
     op = tmp._call_stack[-1][0]
     if op == _IndexedComponent_slice.get_item:
         # If the last attribute of the slice gets an item,
         # change it to delete the item
         tmp._call_stack[-1] = (
             _IndexedComponent_slice.del_item,
             tmp._call_stack[-1][1] )
     elif op == _IndexedComponent_slice.slice_info:
         assert len(tmp._call_stack) == 1
         _iter = self._get_iter(tmp, key)
         try:
             advance_iterator(_iter)
             del _iter._iter_stack[0].component[_iter.get_last_index()]
             return
         except StopIteration:
             raise KeyError("KeyError: %s" % (key,))
     elif op == _IndexedComponent_slice.get_attribute:
         # If the last attribute of the slice retrieves an attribute,
         # change it to delete the attribute
         tmp._call_stack[-1] = (
             _IndexedComponent_slice.del_attribute,
             tmp._call_stack[-1][1] )
     else:
         raise DeveloperError(
             "Unexpected slice _call_stack operation: %s" % op)
     try:
         advance_iterator(self._get_iter(tmp, key))
     except StopIteration:
         pass

    def run_concurrent_tests(self, input_iter):
        """Run tests concurrently."""

        it = iter(input_iter)
        numtests = 0
        try:
            for proc in self.procs:
                self.task_queue.put(advance_iterator(it))
                numtests += 1
        except StopIteration:
            pass
        else:
            try:
                while numtests:
                    result = self.done_queue.get()
                    yield result
                    numtests -= 1
                    if self.stop and result.status == 'FAIL':
                        break
                    self.task_queue.put(advance_iterator(it))
                    numtests += 1
            except StopIteration:
                pass

        for proc in self.procs:
            self.task_queue.put('STOP')

        for i in range(numtests):
            yield self.done_queue.get()

        for proc in self.procs:
            proc.join()

 def _iter(self):
     rlist = []
     self._rdate.sort()
     self._genitem(rlist, iter(self._rdate))
     for gen in [iter(x) for x in self._rrule]:
         self._genitem(rlist, gen)
     rlist.sort()
     exlist = []
     self._exdate.sort()
     self._genitem(exlist, iter(self._exdate))
     for gen in [iter(x) for x in self._exrule]:
         self._genitem(exlist, gen)
     exlist.sort()
     lastdt = None
     total = 0
     while rlist:
         ritem = rlist[0]
         if not lastdt or lastdt != ritem.dt:
             while exlist and exlist[0] < ritem:
                 advance_iterator(exlist[0])
                 exlist.sort()
             if not exlist or ritem != exlist[0]:
                 total += 1
                 yield ritem.dt
             lastdt = ritem.dt
         advance_iterator(ritem)
         rlist.sort()
     self._len = total

 def __setitem__(self, key, val):
     tmp = self._slice.duplicate()
     op = tmp._call_stack[-1][0]
     if op == _IndexedComponent_slice.get_item:
         tmp._call_stack[-1] = (
             _IndexedComponent_slice.set_item,
             tmp._call_stack[-1][1],
             val )
     elif op == _IndexedComponent_slice.slice_info:
         tmp._call_stack[-1] = (
             _IndexedComponent_slice.set_item,
             tmp._call_stack[-1][1],
             val )
     elif op == _IndexedComponent_slice.get_attribute:
         tmp._call_stack[-1] = (
             _IndexedComponent_slice.set_attribute,
             tmp._call_stack[-1][1],
             val )
     else:
         raise DeveloperError(
             "Unexpected slice _call_stack operation: %s" % op)
     try:
         advance_iterator(self._get_iter(tmp, key, get_if_not_present=True))
     except StopIteration:
         pass

 def test_imap_unordered(self):
     p = pool.Pool(1)
     it = p.imap_unordered(divide_by, [1, 0, 2])
     self.assertEqual(six.advance_iterator(it), 1.0)
     self.assertRaises(ZeroDivisionError, six.advance_iterator, it)
     self.assertEqual(six.advance_iterator(it), 0.5)
     self.assertRaises(StopIteration, six.advance_iterator, it)

    def test_iter_one_item(self):
        self.r.request = Mock()
        self.r.request.return_value = Mock(), {'meta': {'key': 'foos', 'next_page_url': None}, 'foos': [{'sid': '123'}]}

        items = self.r.iter()
        advance_iterator(items)

        self.assertRaises(StopIteration, advance_iterator, items)

    def testIterOneItem(self):
        self.r.request = Mock()
        self.r.request.return_value = Mock(), {self.r.key: [{'sid': 'foo'}]}

        items = self.r.iter()
        advance_iterator(items)

        self.assertRaises(StopIteration, advance_iterator, items)

 def test_imap_unordered_gc(self):
     it = self.pool.imap_unordered(sqr, range(10))
     result = []
     for i in range(10):
         result.append(six.advance_iterator(it))
         gc.collect()
     self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
     self.assertEqual(sorted(result), [x * x for x in range(10)])

def test_iterator():
    class myiter(six.Iterator):
        def __next__(self):
            return 13
    assert six.advance_iterator(myiter()) == 13
    class myitersub(myiter):
        def __next__(self):
            return 14
    assert six.advance_iterator(myitersub()) == 14

 def popitem(self, last=True):
     if not self:
         raise KeyError('dictionary is empty')
     if last:
         key = six.advance_iterator(reversed(self))
     else:
         key = six.advance_iterator(iter(self))
     value = self.pop(key)
     return key, value

    def sample(self, cursor):
        """Extract records randomly from the database.
        Continue until the target proportion of the items have been
        extracted, or until `min_items` if this is larger.
        If `max_items` is non-negative, do not extract more than these.

        This function is a generator, yielding items incrementally.

        :param cursor: Cursor to sample
        :type cursor: pymongo.cursor.Cursor
        :return: yields each item
        :rtype: dict
        :raise: ValueError, if max_items is valid and less than `min_items`
                or if target collection is empty
        """
        count = cursor.count()

        # special case: empty collection
        if count == 0:
            self._empty = True
            raise ValueError("Empty collection")

        # special case: entire collection
        if self.p >= 1 and self.max_items <= 0:
            for item in cursor:
                yield item
            return

        # calculate target number of items to select
        if self.max_items <= 0:
            n_target = max(self.min_items, self.p * count)
        else:
            if self.p <= 0:
                n_target = max(self.min_items, self.max_items)
            else:
                n_target = max(self.min_items, min(self.max_items, self.p * count))
        if n_target == 0:
            raise ValueError("No items requested")

        # select first `n_target` items that pop up with
        # probability self.p
        # This is actually biased to items at the beginning
        # of the file if n_target is smaller than (p * count),
        n = 0
        while n < n_target:
            try:
                item = six.advance_iterator(cursor)
            except StopIteration:
                # need to keep looping through data until
                # we get all our items!
                cursor.rewind()
                item = six.advance_iterator(cursor)
            if self._keep():
                yield item
                n += 1

def _skip_ws(tok, stream, state, internal):
    while tok.type  in ["NEWLINE", "WS"]:
        if tok.type == "NEWLINE" and len(internal.words_stack) == 0:
            nxt = stream.peek()
            if not nxt.type == "INDENT":
                state = "SCANNING_FIELD_ID"
            else:
                tok = six.advance_iterator(stream)
            return tok, state
        tok = six.advance_iterator(stream)
    return tok, state

    def test_imap(self):
        it = self.pool.imap(sqr, range(10))
        self.assertEqual(list(it), list(map(squared, range(10))))

        it = self.pool.imap(sqr, range(10))
        for i in range(10):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

        it = self.pool.imap(sqr, range(1000))
        for i in range(1000):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

 def checkIteratorParallel(self):
     self._dostore()
     self._dostore()
     iter1 = self._storage.iterator()
     iter2 = self._storage.iterator()
     txn_info1 = six.advance_iterator(iter1)
     txn_info2 = six.advance_iterator(iter2)
     self.assertEqual(txn_info1.tid, txn_info2.tid)
     txn_info1 = six.advance_iterator(iter1)
     txn_info2 = six.advance_iterator(iter2)
     self.assertEqual(txn_info1.tid, txn_info2.tid)
     self.assertRaises(StopIteration, next, iter1)
     self.assertRaises(StopIteration, next, iter2)

def skip_until_eol(stream, t):
    try:
        prev = stream.previous()
    except ValueError:
        prev = None
    while t.type != "NEWLINE":
        t = six.advance_iterator(stream)
    # FIXME: ideally, we would like to remove EOL for comments which span the
    # full line, but we need access to the token before the comment delimiter
    # to do so, as we don't want to remove EOL for inline commeng (e.g. 'foo #
    # comment')
    if prev and t.type == "NEWLINE" and prev.type in ('NEWLINE', 'INDENT'):
        t = six.advance_iterator(stream)
    return t

    def checkIteratorGCStorageTPCAborting(self):
        # The odd little jig we do below arises from the fact that the
        # CS iterator may not be constructed right away if the CS is wrapped.
        # We need to actually do some iteration to get the iterator created.
        # We do a store to make sure the iterator isn't exhausted right away.
        self._dostore()
        six.advance_iterator(self._storage.iterator())

        iid = list(self._storage._iterator_ids)[0]

        t = transaction.Transaction()
        self._storage.tpc_begin(t)
        self._storage.tpc_abort(t)
        self.assertEquals(0, len(self._storage._iterator_ids))
        self.assertRaises(KeyError, self._storage._server.iterator_next, iid)

 def __contains__(self, key):
     try:
         advance_iterator(self._get_iter(self._slice, key))
         return True
     except (StopIteration, KeyError):
         return False
     except SliceEllipsisLookupError:
         if type(key) is tuple and len(key) == 1:
             key = key[0]
         # Brute force (linear time) lookup
         _iter = iter(self._slice)
         for item in _iter:
             if _iter.get_last_index_wildcards() == key:
                 return True
         return False

def test_PushbackAdapter():
    it = PushbackAdapter(iter([1, 2, 3, 4]))
    assert it.has_more()
    assert six.advance_iterator(it) == 1
    it.push_back(0)
    assert six.advance_iterator(it) == 0
    assert six.advance_iterator(it) == 2
    assert it.peek() == 3
    it.push_back(10)
    assert it.peek() == 10
    it.push_back(20)
    assert it.peek() == 20
    assert it.has_more()
    assert list(it) == [20, 10, 3, 4]
    assert not it.has_more()

    def checkIteratorExhaustionStorage(self):
        # Test the storage's garbage collection mechanism.
        self._dostore()
        iterator = self._storage.iterator()

        # At this point, a wrapping iterator might not have called the CS
        # iterator yet. We'll consume one item to make sure this happens.
        six.advance_iterator(iterator)
        self.assertEqual(1, len(self._storage._iterator_ids))
        iid = list(self._storage._iterator_ids)[0]
        self.assertEqual([], list(iterator))
        self.assertEqual(0, len(self._storage._iterator_ids))

        # The iterator has run through, so the server has already disposed it.
        self.assertRaises(KeyError, self._storage._call, 'iterator_next', iid)