def _get_lines(self, file1, file2, lines, file1_dest, file2_dest):
        """
        Given two files to open and a sorted list of lines to get, open the files,
        retrieves the desired lines, and dumps them to specified file locations.
        """
        lines = deque(lines)
        buf1, buf2 = [], []
        line_counter, target_line = 0, lines.popleft()

        for f1_line, f2_line in zip(open(file1, 'r'), open(file2, 'r')):
            if target_line == line_counter:
                buf1.append(f1_line.strip())
                buf2.append(f2_line.strip())

                if asizeof.asizeof(buf1) + asizeof.asizeof(buf2) > \
                    self.mem_limit:
                    self._dump_bufs_to( [file1_dest, file2_dest],
                                        [buf1, buf2])

                if len(lines) != 0:
                    target_line = lines.popleft()
                else:
                    break
            line_counter += 1

        self._dump_bufs_to( [file1_dest, file2_dest],
                            [buf1, buf2])

 def get_memory_usage(self):
     """
     Returns the sizes (in bytes) of the four main models that the classifier keeps in memory.
     """
     with self.lock:
         return asizeof.asizeof(self.hc), asizeof.asizeof(self.htc), asizeof.asizeof(self.tc), \
             asizeof.asizeof(self.thc)

def test_slots_being_used():
    """
    The class is really using __slots__.
    """
    non_slot_instance = C1(x=1, y="test")
    slot_instance = C1Slots(x=1, y="test")

    assert "__dict__" not in dir(slot_instance)
    assert "__slots__" in dir(slot_instance)

    assert "__dict__" in dir(non_slot_instance)
    assert "__slots__" not in dir(non_slot_instance)

    assert set(["x", "y"]) == set(slot_instance.__slots__)

    if has_pympler:
        assert asizeof(slot_instance) < asizeof(non_slot_instance)

    non_slot_instance.t = "test"
    with pytest.raises(AttributeError):
        slot_instance.t = "test"

    assert 1 == non_slot_instance.method()
    assert 1 == slot_instance.method()

    assert attr.fields(C1Slots) == attr.fields(C1)
    assert attr.asdict(slot_instance) == attr.asdict(non_slot_instance)

    def cleanse(self, src_lang_file, tar_lang_file):
        """
        Cleans the file provided by lowercasing all words and ensuring each line in
        the text file is within min_len and max_len. Operates on two streams
        simultaneously in order to keep line to line correspondence
        """
        self._validate_file(src_lang_file), self._validate_file(tar_lang_file)
        src_dest_file = self.destdir + utilities.strip_filename_from_path(src_lang_file) + ".cleansed"
        tar_dest_file = self.destdir + utilities.strip_filename_from_path(tar_lang_file) + ".cleansed"

        if utilities.files_exist([src_dest_file, tar_dest_file]):
            return
        else:
            utilities.wipe_files([src_dest_file, tar_dest_file])
        self._print("""Cleaning data.  Ensuring uniformity of data...""")

        src_buf, tar_buf = [], []
        for src_line, tar_line in zip(open(src_lang_file), open(tar_lang_file)):
            src_line = src_line.lower().split()
            tar_line = tar_line.lower().split()

            if len(src_line) > self.min_len and len(src_line) < self.max_len and \
                len(tar_line) > self.min_len and len(tar_line) < self.max_len:
                src_buf.append(' '.join(src_line))
                tar_buf.append(' '.join(tar_line))

            if asizeof.asizeof(src_buf) + asizeof.asizeof(tar_buf) > self.mem_limit:
                self._dump_bufs_to( [src_dest_file, tar_dest_file],
                                    [src_buf, tar_buf])

        self._dump_bufs_to([src_dest_file, tar_dest_file], [src_buf, tar_buf])
        self._print("Done\n")

def test_copy_features_does_not_copy_entityset(es):
    agg = Sum(es['log']['value'], es['sessions'])
    agg_where = Sum(es['log']['value'], es['sessions'],
                    where=IdentityFeature(es['log']['value']) == 2)
    agg_use_previous = Sum(es['log']['value'], es['sessions'],
                           use_previous='4 days')
    agg_use_previous_where = Sum(es['log']['value'], es['sessions'],
                                 where=IdentityFeature(es['log']['value']) == 2,
                                 use_previous='4 days')
    features = [agg, agg_where, agg_use_previous, agg_use_previous_where]
    in_memory_size = asizeof(locals())
    copied = [f.copy() for f in features]
    new_in_memory_size = asizeof(locals())
    assert new_in_memory_size < 2 * in_memory_size

    for f, c in zip(features, copied):
        assert f.entityset
        assert c.entityset
        assert id(f.entityset) == id(c.entityset)
        if f.where:
            assert c.where
            assert id(f.where.entityset) == id(c.where.entityset)
        for bf, bf_c in zip(f.base_features, c.base_features):
            assert id(bf.entityset) == id(bf_c.entityset)
            if bf.where:
                assert bf_c.where
                assert id(bf.where.entityset) == id(bf_c.where.entityset)

 def process_response(self, request, response):
         req = request.META['PATH_INFO']
         if req.find('static') == -1 and req.find('media') == -1:
                 print req
                 self.end_objects = muppy.get_objects()
                 sum_start = summary.summarize(self.start_objects)
                 sum_end = summary.summarize(self.end_objects)
                 diff = summary.get_diff(sum_start, sum_end)
                 summary.print_(diff)
                 #print '~~~~~~~~~'
                 #cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \
                         #str_func=output_function)
                 #cb.print_tree()
                 print '~~~~~~~~~'
                 a = asizeof(response)
                 print 'Total size of response object in kB: %s' % \
                     str(a / 1024.0)
                 print '~~~~~~~~~'
                 a = asizeof(self.end_objects)
                 print 'Total size of end_objects in MB: %s' % \
                     str(a / 1048576.0)
                 b = asizeof(self.start_objects)
                 print 'Total size of start_objects in MB: %s' % \
                     str(b / 1048576.0)
                 print '~~~~~~~~~'
         return response

 def push(self, msg):
     serialized_msg = pickle.dumps(msg)
     from pympler.asizeof import asizeof
     print('unpickled: {}, pickled: {}'.format(
         asizeof(msg),
         asizeof(serialized_msg)
     ))
     self.output(serialized_msg)

    def test_globals(self):
        '''Test globals examples'''
        self._printf('%sasizeof(%s, limit=%s, code=%s) ... %s', os.linesep, 'globals()', 'MAX', False, '-glob[als]')
        asizeof.asizeof(globals(), limit=self.MAX, code=False, stats=1)
        self._print_functions(globals(), 'globals()', opt='-glob[als]')

        self._printf('%sasizesof(%s, limit=%s, code=%s) ... %s', os.linesep, 'globals(), locals()', 'MAX', False, '-glob[als]')
        asizeof.asizesof(globals(), locals(), limit=self.MAX, code=False, stats=1)
        asizeof.asized(globals(), align=0, detail=self.MAX, limit=self.MAX, code=False, stats=1)

    def test_methods(self):
        '''Test sizing methods and functions
        '''
        def foo():
            pass

        s1 = asizeof.asizeof(self.test_methods, code=True)
        s2 = asizeof.asizeof(TypesTest.test_methods, code=True)
        s3 = asizeof.asizeof(foo, code=True)

 def test_adict(self):
     '''Test asizeof.adict()
     '''
     pdict = PseudoDict()
     size1 = asizeof.asizeof(pdict)
     asizeof.adict(PseudoDict)
     size2 = asizeof.asizeof(pdict)
     # TODO: come up with useful assertions
     self.assertEqual(size1, size2)

def run(dicp="~/dev/kaggle/fb5/pdic.map", datap="~/dev/kaggle/fb5/train.tab", lr=1., numbats=100, epochs=10):
    dic, revdic = loaddict(expanduser(dicp))
    print len(dic)
    traindata, golddata = loaddata(expanduser(datap), top=10000)
    print asizeof(traindata), golddata.dtype
    m = SpatialEmb(dim=len(dic))

    m.train([traindata], golddata).adagrad(lr=lr).cross_entropy()\
        .split_validate(splits=100, random=True).cross_entropy().accuracy()\
        .train(numbats, epochs)

 def test_asizer(self):
     '''Test Asizer properties.
     '''
     sizer = asizeof.Asizer()
     obj = 'unladen swallow'
     mutable = [obj]
     sizer.asizeof(obj)
     self.assertEqual(sizer.total, asizeof.asizeof(obj))
     sizer.asizeof(mutable, mutable)
     self.assertEqual(sizer.duplicate, 1)
     self.assertEqual(sizer.total, asizeof.asizeof(obj, mutable))

 def add_results_data(self, results):
     if SIZE_CONTROL:
         if not self.MEM_LIMIT:
             mem_size = asizeof(self.current_task.results)
             add_size = asizeof(results)
             if (mem_size + add_size) < 15000000:
                 self._add_results(results)
             else:
                 self.MEM_LIMIT = True
     else:
         self._add_results(results)

 def getSizeOfMgrs(self):
     """ get size of object """
     appGlobal = config['pylons.app_globals']
     
     result = {}
     result['threadmgr'] = asizeof(appGlobal.threadMgr)
     result['packagemgr'] = asizeof(appGlobal.packageMgr)
     result['montior'] = asizeof(appGlobal.agentMonitor)
     result['all'] = asizeof(appGlobal)
     
     return doneResult(request, response, result = result, controller = self)

    def test_private_slots(self):
        class PrivateSlot(object):
            __slots__ = ('__data',)
            def __init__(self, data):
                self.__data = data

        data = [42] * 100
        container = PrivateSlot(data)
        size1 = asizeof.asizeof(container)
        size2 = asizeof.asizeof(data)
        self.assertTrue(size1 > size2, (size1, size2))

    def test_asizeof(self):
        '''Test asizeof.asizeof()
        '''
        self.assertEqual(asizeof.asizeof(), 0)

        objs = [Foo(42), ThinFoo("spam"), OldFoo(67)]
        total = asizeof.asizeof(*objs)
        sizes = list(asizeof.asizesof(*objs))
        sum = 0
        for sz in sizes:
            sum += sz
        self.assertEqual(total, sum, (total, sum))

 def dumpMonitorValues(self):
     ''' dump all monitor values as json string '''
     result = {}
     for (service, mname) in self.__monitorValues:
         key = '%s.%s' % (service, mname)
         result[key] = self.__monitorValues[(service, mname)]
     result['values'] = asizeof(self.__monitorValues)
     result['tasks'] = asizeof(self.__monitorTasks)
     result['messages'] = asizeof(self.__monitorMessages)
     result['tags'] = asizeof(self.__monitorTags)
     result['messagekeys'] = '%s' % self.__monitorMessages
     return result

    def test_closure(self):
        '''Test sizing closures.
        '''
        def outer(x):
            def inner():
                return x
            return inner

        data = [1] * 1000
        closure = outer(data)
        size_closure = asizeof.asizeof(closure, code=True)
        size_data = asizeof.asizeof(data)
        self.assertTrue(size_closure >= size_data, (size_closure, size_data))

    def split_train_tune_test(self, src_file, src_piv_file, piv_tar_file, tar_file,
        train_split, test_split):
        """
        Splits the full datafiles into test, tune, and train sets.
        Receives 4 files as parameters and 2 decimals indicating the percentage of
        data to be used as train, tune, and test data. If line 1 in src langs is
        in test, then line 1 in tar langs will also be in test. Etc.
        """
        utilities.make_dir(self.traindir)
        utilities.make_dir(self.tunedir)
        utilities.make_dir(self.testdir)

        self._validate_file(src_file), self._validate_file(src_piv_file)
        self._validate_file(piv_tar_file), self._validate_file(tar_file)
        assert train_split + test_split <= 1 , "Invalid size for train, tune, and test splits"

        train_files, tune_files, test_files = self._ttt_filenames(src_file, src_piv_file, piv_tar_file, tar_file)
        if utilities.ttt_files_exist(train_files, tune_files, test_files):
            return
        else:
            utilities.ttt_wipe_files(train_files, tune_files, test_files)

        self._print("""Splitting data into train, tune, and test sets...""")
        train, tune, test = [[] ,[], [], []],  [[], [], [], []], [[], [], [], []]
        for src_line, src_piv_line, piv_tar_line, tar_line in \
            zip_longest(open(src_file), open(src_piv_file), open(piv_tar_file), open(tar_file)):

            x = numpy.random.sample()
            if x < train_split:
                self._add_line_to(train[0], src_line)
                self._add_line_to(train[1], src_piv_line)
                self._add_line_to(train[2], piv_tar_line)
                self._add_line_to(train[3], tar_line)
            elif x >= train_split and x < train_split + test_split:
                self._add_line_to(tune[0], src_line)
                self._add_line_to(tune[1], src_piv_line)
                self._add_line_to(tune[2], piv_tar_line)
                self._add_line_to(tune[3], tar_line)
            else:
                self._add_line_to(test[0], src_line)
                self._add_line_to(test[1], src_piv_line)
                self._add_line_to(test[2], piv_tar_line)
                self._add_line_to(test[3], tar_line)

            if asizeof.asizeof(train) + asizeof.asizeof(tune) + \
                asizeof.asizeof(test) > self.mem_limit:
                self._dump_ttt_bufs_to(train, tune, test, train_files, tune_files, test_files)

        self._dump_ttt_bufs_to(train, tune, test, train_files, tune_files, test_files)
        self._print("Done\n")

 def test_weakref(self):
     '''Test sizing weak references.
     '''
     alive = Foo('alive')
     aref = weakref.ref(alive)
     dead = Foo('dead')
     dref = weakref.ref(dead)
     del dead
     aref_size = asizeof.asizeof(aref)
     self.assertTrue(aref_size > asizeof.asizeof(alive), aref_size)
     refs = asizeof.named_refs(aref)
     # TODO: Should a weakref return ('ref', obj)?
     dref_size = asizeof.asizeof(dref)
     self.assertTrue(dref_size > 0, dref_size)
     self.assertNotEqual(dref_size, aref_size)
     refs = asizeof.named_refs(dref)