def attach_handle_on_suspstack(self, handle):
     s = self.suspstack
     self.suspstack = NULL_SUSPSTACK
     ll_assert(bool(s.anchor), "s.anchor should not be null")
     s.handle = handle
     llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(s))
     return s

def get_result_suspstack(h):
    # Now we are in the target, after the switch() or the new().
    # Note that this whole module was carefully written in such a way as
    # not to invoke pushing/popping things off the shadowstack at
    # unexpected moments...
    oldsuspstack = gcrootfinder.oldsuspstack
    newsuspstack = gcrootfinder.newsuspstack
    gcrootfinder.oldsuspstack = NULL_SUSPSTACK
    gcrootfinder.newsuspstack = NULL_SUSPSTACK
    if not h:
        raise MemoryError
    # We still have the old shadowstack active at this point; save it
    # away, and restore the new one
    if oldsuspstack:
        ll_assert(not _c.is_empty_handle(h), "unexpected empty stacklet handle")
        h = llmemory.cast_ptr_to_adr(h)
        llop.gc_save_current_state_away(lltype.Void, oldsuspstack, h)
    else:
        ll_assert(_c.is_empty_handle(h), "unexpected non-empty stacklet handle")
        llop.gc_forget_current_state(lltype.Void)
    #
    llop.gc_restore_state_from(lltype.Void, newsuspstack)
    #
    # From this point on, 'newsuspstack' is consumed and done, its
    # shadow stack installed as the current one.  It should not be
    # used any more.  For performance, we avoid it being deallocated
    # by letting it be reused on the next switch.
    gcrootfinder.oldsuspstack = newsuspstack
    # Return.
    return oldsuspstack

 def _really_force(self):
     if self.source_op is None:
         # this case should not occur; I only managed to get it once
         # in pypy-c-jit and couldn't reproduce it.  The point is
         # that it relies on optimizefindnode.py computing exactly
         # the right level of specialization, and it seems that there
         # is still a corner case where it gets too specialized for
         # optimizeopt.py.  Let's not crash in release-built
         # pypy-c-jit's.  XXX find out when
         from pypy.rlib.debug import ll_assert
         ll_assert(False, "_really_force: source_op is None")
         raise InvalidLoop
     #
     newoperations = self.optimizer.newoperations
     newoperations.append(self.source_op)
     self.box = box = self.source_op.result
     #
     iteritems = self._fields.iteritems()
     if not we_are_translated(): #random order is fine, except for tests
         iteritems = list(iteritems)
         iteritems.sort(key = lambda (x,y): x.sort_key())
     for ofs, value in iteritems:
         if value.is_null():
             continue
         subbox = value.force_box()
         op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                           descr=ofs)
         newoperations.append(op)
     self._fields = None

 def compute_alive_objects(self):
     fromaddr = self.space
     addraftercollect = self.space
     num = 1
     while fromaddr < self.free:
         size_gc_header = self.gcheaderbuilder.size_gc_header
         tid = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)).tid
         obj = fromaddr + size_gc_header
         objsize = self.get_size(obj)
         objtotalsize = size_gc_header + objsize
         if self.marked(obj):
             copy_has_hash_field = ((tid & GCFLAG_HASHFIELD) != 0 or
                                    ((tid & GCFLAG_HASHTAKEN) != 0 and
                                     addraftercollect < fromaddr))
             addraftercollect += raw_malloc_usage(objtotalsize)
             if copy_has_hash_field:
                 addraftercollect += llmemory.sizeof(lltype.Signed)
         num += 1
         fromaddr += objtotalsize
         if tid & GCFLAG_HASHFIELD:
             fromaddr += llmemory.sizeof(lltype.Signed)
     ll_assert(addraftercollect <= fromaddr,
               "markcompactcollect() is trying to increase memory usage")
     self.totalsize_of_objs = addraftercollect - self.space
     return num

 def replay(self, label, kinds):
     builder = LLBuilder(self, label.g, llimpl.nullblock)
     args_gv = builder._newblock(kinds)
     ll_assert(self.currently_writing is None,
                  "replay: currently_writing")
     self.currently_writing = builder
     return builder, args_gv

def ll_extend_with_str_slice(lst, s, getstrlen, getstritem, slice):
    start = slice.start
    stop = slice.stop
    len1 = lst.ll_length()
    len2 = getstrlen(s)
    ll_assert(start >= 0, "unexpectedly negative str slice start")
    ll_assert(start <= len2, "str slice start larger than str length")
    if stop > len2:
        stop = len2
    count2 = stop - start
    assert count2 >= 0, "str slice stop smaller than start"
    try:
        newlength = ovfcheck(len1 + count2)
    except OverflowError:
        raise MemoryError
    lst._ll_resize_ge(newlength)
    i = start
    j = len1
    while i < stop:
        c = getstritem(s, i)
        if listItemType(lst) is UniChar:
            c = unichr(ord(c))
        lst.ll_setitem_fast(j, c)
        i += 1
        j += 1

 def collect_nursery(self):
     if self.nursery_size > self.top_of_space - self.free:
         # the semispace is running out, do a full collect
         self.obtain_free_space(self.nursery_size)
         ll_assert(self.nursery_size <= self.top_of_space - self.free,
                      "obtain_free_space failed to do its job")
     if self.nursery:
         if DEBUG_PRINT:
             llop.debug_print(lltype.Void, "minor collect")
         # a nursery-only collection
         scan = beginning = self.free
         self.collect_oldrefs_to_nursery()
         self.collect_roots_in_nursery()
         scan = self.scan_objects_just_copied_out_of_nursery(scan)
         # at this point, all static and old objects have got their
         # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
         self.notify_objects_just_moved()
         # mark the nursery as free and fill it with zeroes again
         llarena.arena_reset(self.nursery, self.nursery_size, True)
         if DEBUG_PRINT:
             llop.debug_print(lltype.Void, "percent survived:", float(scan - beginning) / self.nursery_size)
     else:
         # no nursery - this occurs after a full collect, triggered either
         # just above or by some previous non-nursery-based allocation.
         # Grab a piece of the current space for the nursery.
         self.nursery = self.free
         self.nursery_top = self.nursery + self.nursery_size
         self.free = self.nursery_top
     self.nursery_free = self.nursery
     return self.nursery_free

 def ll_iter_broadcast_to_shape(ITER, ao, target_ao):
     "iterate over <ao> but broadcast to the shape of <target_ao>"
     ll_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim")
     delta = j = ndim - ao.ndim
     shape = target_ao.shape
     for i in range(ao.ndim):
         if ao.shape[i] != 1 and ao.shape[i] != shape[j]:
             raise Exception("array is not broadcastable to correct shape")
         j += 1
     it = malloc(ITER)
     it.size = ll_mul_list(target_ao.shape, ndim)
     it.nd_m1 = ndim - 1
     #it.factors[ndim-1] = 1
     for i in unroll_ndim:
         it.dims_m1[i] = shape[i]-1
         k = i - delta
         if k<0 or ao.shape[k] != shape[i]:
             #it.contiguous = False
             it.strides[i] = 0
         else:
             it.strides[i] = ao.strides[k]
         it.backstrides[i] = it.strides[i] * it.dims_m1[i]
         #if i > 0:
             #it.factors[ndim-i-1] = it.factors[nd-i]*shape[ndim-i]
     it.ll_reset(ao.dataptr)
     return it

def ll_unerase_int(gcref):
    from pypy.rpython.lltypesystem.lloperation import llop
    from pypy.rlib.debug import ll_assert

    x = llop.cast_ptr_to_int(lltype.Signed, gcref)
    ll_assert((x & 1) != 0, "unerased_int(): not an integer")
    return x >> 1

 def get_header_forwarded_addr(self, obj):
     tid = self.header_forwarded(obj).tid
     ll_assert(tid & GCFLAG_MARKBIT != 0, "dying object is not forwarded")
     GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3)
     res = self.base_forwarding_addr + (tid & GCFLAG_MASK) + self.gcheaderbuilder.size_gc_header
     ll_assert(res < self.finaladdr, "forwarded address >= self.finaladdr")
     return res

 def genraisingop1(self, opname, gv_arg):
     ll_assert(self.rgenop.currently_writing is self,
                  "genraisingop1: bad currently_writing")
     gv_res = LLVar(llimpl.genop(self.b, opname, [gv_arg], llimpl.guess))
     gv_exc = LLVar(llimpl.genop(self.b, "check_and_clear_exc", [],
                                 gv_Bool.v))
     return gv_res, gv_exc

    def make_a_nonmoving_copy(self, obj, objsize):
        # NB. the object can have a finalizer or be a weakref, but
        # it's not an issue.
        totalsize = self.size_gc_header() + objsize
        newaddr = self.allocate_external_object(totalsize)
        if not newaddr:
            return llmemory.NULL   # can't raise MemoryError during a collect()
        if self.config.gcconfig.debugprint:
            self._nonmoving_copy_count += 1
            self._nonmoving_copy_size += raw_malloc_usage(totalsize)

        llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize)
        newobj = newaddr + self.size_gc_header()
        hdr = self.header(newobj)
        hdr.tid |= self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS
        # GCFLAG_UNVISITED is not set
        # GCFLAG_NO_HEAP_PTRS is not set either, conservatively.  It may be
        # set by the next collection's collect_last_generation_roots().
        # This old object is immediately put at generation 3.
        ll_assert(self.is_last_generation(newobj),
                  "make_a_nonmoving_copy: object too young")
        self.gen3_rawmalloced_objects.append(newobj)
        self.last_generation_root_objects.append(newobj)
        self.rawmalloced_objects_to_trace.append(newobj)   # visit me
        return newobj

    def make_a_nonmoving_copy(self, obj, objsize):
        # NB. the object can have a finalizer or be a weakref, but
        # it's not an issue.
        totalsize = self.size_gc_header() + objsize
        tid = self.header(obj).tid
        if tid & GCFLAG_HASHMASK:
            totalsize_incl_hash = totalsize + llmemory.sizeof(lltype.Signed)
        else:
            totalsize_incl_hash = totalsize
        newaddr = self.allocate_external_object(totalsize_incl_hash)
        if not newaddr:
            return llmemory.NULL   # can't raise MemoryError during a collect()
        self._nonmoving_copy_count += 1
        self._nonmoving_copy_size += raw_malloc_usage(totalsize)

        llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize)
        if tid & GCFLAG_HASHMASK:
            hash = self._get_object_hash(obj, objsize, tid)
            (newaddr + totalsize).signed[0] = hash
            tid |= GC_HASH_HASFIELD
        #
        # GCFLAG_UNVISITED is not set
        # GCFLAG_NO_HEAP_PTRS is not set either, conservatively.  It may be
        # set by the next collection's collect_last_generation_roots().
        # This old object is immediately put at generation 3.
        newobj = newaddr + self.size_gc_header()
        hdr = self.header(newobj)
        hdr.tid = tid | self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS
        ll_assert(self.is_last_generation(newobj),
                  "make_a_nonmoving_copy: object too young")
        self.gen3_rawmalloced_objects.append(newobj)
        self.last_generation_root_objects.append(newobj)
        self.rawmalloced_objects_to_trace.append(newobj)   # visit me
        return newobj

 def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow):
     size_gc_header = self.size_gc_header()
     addr = llmemory.cast_ptr_to_adr(ptr)
     ll_assert(self.header(addr).tid & GCFLAG_EXTERNAL,
               "realloc() on a non-external object")
     nonvarsize = size_gc_header + fixedsize
     try:
         varsize = ovfcheck(itemsize * newlength)
         tot_size = ovfcheck(nonvarsize + varsize)
     except OverflowError:
         raise MemoryError()
     oldlength = (addr + lengthofs).signed[0]
     old_tot_size = size_gc_header + fixedsize + oldlength * itemsize
     source_addr = addr - size_gc_header
     self.gen2_resizable_objects.remove(addr)
     if grow:
         result = llop.raw_realloc_grow(llmemory.Address, source_addr,
                                        old_tot_size, tot_size)
     else:
         result = llop.raw_realloc_shrink(llmemory.Address, source_addr,
                                          old_tot_size, tot_size)
     if not result:
         self.gen2_resizable_objects.append(addr)
         raise MemoryError()
     if grow:
         self.gen2_resizable_objects.append(result + size_gc_header)
     else:
         self.gen2_rawmalloced_objects.append(result + size_gc_header)
     self._check_rawsize_alloced(raw_malloc_usage(tot_size) -
                                 raw_malloc_usage(old_tot_size),
                                 can_collect = not grow)
     (result + size_gc_header + lengthofs).signed[0] = newlength
     return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF)

 def restore_state_from(self, shadowstackref):
     ll_assert(bool(shadowstackref.base), "empty shadowstackref!")
     ll_assert(shadowstackref.base <= shadowstackref.top,
               "restore_state_from: broken shadowstack")
     self.gcdata.root_stack_base = shadowstackref.base
     self.gcdata.root_stack_top  = shadowstackref.top
     self._cleanup(shadowstackref)

 def trace_partial(self, obj, start, stop, callback, arg):
     """Like trace(), but only walk the array part, for indices in
     range(start, stop).  Must only be called if has_gcptr_in_varsize().
     """
     length = stop - start
     typeid = self.get_type_id(obj)
     if self.is_gcarrayofgcptr(typeid):
         # a performance shortcut for GcArray(gcptr)
         item = obj + llmemory.gcarrayofptr_itemsoffset
         item += llmemory.gcarrayofptr_singleitemoffset * start
         while length > 0:
             if self.points_to_valid_gc_object(item):
                 callback(item, arg)
             item += llmemory.gcarrayofptr_singleitemoffset
             length -= 1
         return
     ll_assert(self.has_gcptr_in_varsize(typeid),
               "trace_partial() on object without has_gcptr_in_varsize()")
     item = obj + self.varsize_offset_to_variable_part(typeid)
     offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
     itemlength = self.varsize_item_sizes(typeid)
     item += itemlength * start
     while length > 0:
         j = 0
         while j < len(offsets):
             itemobj = item + offsets[j]
             if self.points_to_valid_gc_object(itemobj):
                 callback(itemobj, arg)
             j += 1
         item += itemlength
         length -= 1

 def setup_root_walker(self):
     stackbase = self.allocate_stack()
     ll_assert(bool(stackbase), "could not allocate root stack")
     self.gcdata.root_stack_top  = stackbase
     self.gcdata.root_stack_base = stackbase
     if self.thread_setup is not None:
         self.thread_setup()

 def ll_compress(fnptr):
     for c, p in unroll_table:
         if fnptr == p:
             return c
     else:
         ll_assert(fnptr == last_p, "unexpected function pointer")
         return last_c

 def genop_ptr_ne(self, gv_PTRTYPE, gv_ptr1, gv_ptr2):
     ll_assert(self.rgenop.currently_writing is self,
                  "genop_ptr_ne: bad currently_writing")
     gv_ptr1 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr1.v)
     gv_ptr2 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr2.v)        
     return LLVar(llimpl.genop(self.b, 'ptr_ne', [gv_ptr1, gv_ptr2],
                               gv_Bool.v))

 def malloc_fixedsize_clear(self, typeid, size, can_collect,
                            has_finalizer=False, contains_weakptr=False):
     if (has_finalizer or not can_collect or
         (raw_malloc_usage(size) > self.lb_young_var_basesize and
          raw_malloc_usage(size) > self.largest_young_fixedsize)):
         # ^^^ we do two size comparisons; the first one appears redundant,
         #     but it can be constant-folded if 'size' is a constant; then
         #     it almost always folds down to False, which kills the
         #     second comparison as well.
         ll_assert(not contains_weakptr, "wrong case for mallocing weakref")
         # "non-simple" case or object too big: don't use the nursery
         return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                   can_collect,
                                                   has_finalizer,
                                                   contains_weakptr)
     size_gc_header = self.gcheaderbuilder.size_gc_header
     totalsize = size_gc_header + size
     result = self.nursery_free
     if raw_malloc_usage(totalsize) > self.nursery_top - result:
         result = self.collect_nursery()
     llarena.arena_reserve(result, totalsize)
     # GCFLAG_NO_YOUNG_PTRS is never set on young objs
     self.init_gc_object(result, typeid, flags=0)
     self.nursery_free = result + totalsize
     if contains_weakptr:
         self.young_objects_with_weakrefs.append(result + size_gc_header)
     return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)