def test_memory_usage_coordinates():
    """
    Watch out for high memory usage on huge spatial files
    """

    ntf = tempfile.NamedTemporaryFile()

    tracemalloc.start()

    snap1 = tracemalloc.take_snapshot()

    # create a "flat" cube
    cube,_ = utilities.generate_gaussian_cube(shape=[1,2000,2000])
    sz = _.dtype.itemsize

    snap1b = tracemalloc.take_snapshot()
    diff = snap1b.compare_to(snap1, 'lineno')
    diffvals = np.array([dd.size_diff for dd in diff])
    # at this point, the generated cube should still exist in memory
    assert diffvals.max()*u.B >= 2000**2*sz*u.B

    del _
    snap2 = tracemalloc.take_snapshot()
    diff = snap2.compare_to(snap1b, 'lineno')
    assert diff[0].size_diff*u.B < -0.3*u.MB

    print(cube)

    # printing the cube should not occupy any more memory
    # (it will allocate a few bytes for the cache, but should *not*
    # load the full 2000x2000 coordinate arrays for RA, Dec
    snap3 = tracemalloc.take_snapshot()
    diff = snap3.compare_to(snap2, 'lineno')
    assert sum([dd.size_diff for dd in diff])*u.B < 100*u.kB

def startMemoryTracing():
    try:
        import tracemalloc  # @UnresolvedImport
    except ImportError:
        pass
    else:
        tracemalloc.start()

    def test_stop_track(self):
        tracemalloc.start()
        tracemalloc.stop()

        with self.assertRaises(RuntimeError):
            self.track()
        self.assertIsNone(self.get_traceback())

    def test_stop_untrack(self):
        tracemalloc.start()
        self.track()

        tracemalloc.stop()
        with self.assertRaises(RuntimeError):
            self.untrack()

    def test_get_traces_intern_traceback(self):
        # dummy wrappers to get more useful and identical frames in the traceback
        def allocate_bytes2(size):
            return allocate_bytes(size)

        def allocate_bytes3(size):
            return allocate_bytes2(size)

        def allocate_bytes4(size):
            return allocate_bytes3(size)

        # Ensure that two identical tracebacks are not duplicated
        tracemalloc.stop()
        tracemalloc.start(4)
        obj_size = 123
        obj1, obj1_traceback = allocate_bytes4(obj_size)
        obj2, obj2_traceback = allocate_bytes4(obj_size)

        traces = tracemalloc._get_traces()

        trace1 = self.find_trace(traces, obj1_traceback)
        trace2 = self.find_trace(traces, obj2_traceback)
        size1, traceback1 = trace1
        size2, traceback2 = trace2
        self.assertEqual(traceback2, traceback1)
        self.assertIs(traceback2, traceback1)

def _start_memory_tracing():
    try:
        import tracemalloc
    except ImportError:
        return

    tracemalloc.start()

    def wrapper(*args, **kwargs):
        import sys
        do_prof, do_tracemalloc, do_traceopen = 3 * [False]
        if len(sys.argv) > 1:
            do_prof = sys.argv[1] == "prof"
            do_tracemalloc = sys.argv[1] == "tracemalloc"
            do_traceopen = sys.argv[1] == "traceopen"

        if do_prof or do_tracemalloc or do_traceopen: sys.argv.pop(1)

        if do_prof:
            print("Entering profiling mode...")
            import pstats, cProfile, tempfile
            prof_file = kwargs.pop("prof_file", None)
            if prof_file is None:
                _, prof_file = tempfile.mkstemp()
                print("Profiling data stored in %s" % prof_file)

            sortby = kwargs.pop("sortby", "time")
            cProfile.runctx("main()", globals(), locals(), prof_file)
            s = pstats.Stats(prof_file)
            s.strip_dirs().sort_stats(sortby).print_stats()
            return 0

        elif do_tracemalloc:
            print("Entering tracemalloc mode...")
            # Requires py3.4
            try:
                import tracemalloc
            except ImportError:
                print("Error while trying to import tracemalloc (requires py3.4)")
                raise SystemExit(1)

            tracemalloc.start()
            retcode = main(*args, **kwargs)
            snapshot = tracemalloc.take_snapshot()
            top_stats = snapshot.statistics('lineno')

            n = min(len(top_stats), 20)
            print("[Top %d]" % n)
            for stat in top_stats[:20]:
                print(stat)

        elif do_traceopen:
            try:
                import psutil
            except ImportError:
                print("traceopen requires psutil module")
                raise SystemExit(1)
            import os
            p = psutil.Process(os.getpid())
            retcode = main(*args, **kwargs)
            print("open_files", p.open_files())

        else:
            retcode = main(*args, **kwargs)

        return retcode

    def peak_monitor_start(self):
        self.peak_monitoring = True

        # start RAM tracing
        tracemalloc.start()

        # this thread samples RAM usage as long as the current epoch of the fit loop is running
        peak_monitor_thread = threading.Thread(target=self.peak_monitor_func)
        peak_monitor_thread.daemon = True
        peak_monitor_thread.start()

    def check_track(self, release_gil):
        nframe = 5
        tracemalloc.start(nframe)

        size = tracemalloc.get_traced_memory()[0]

        frames = self.track(release_gil, nframe)
        self.assertEqual(self.get_traceback(),
                         tracemalloc.Traceback(frames))

        self.assertEqual(self.get_traced_memory(), self.size)

 def __init__(self, interval=60, n_frames=None):
     self.snapshot_q = multiprocessing.Queue()
     self.interval = interval
     self.n_frames = n_frames
     if self.n_frames:
         tracemalloc.start(self.n_frames)
     else:
         tracemalloc.start()
     self.counter = 1
     logger.info('Tracemalloc started')
     super(TakeSnapshot, self).__init__()

def main():
    # Parse command line options
    parser = argparse.ArgumentParser()
    parser.add_argument('input_file', help='input ninja file')
    parser.add_argument('--encoding', default='utf-8',
                        help='ninja file encoding')
    parser.add_argument('--ninja-deps', help='.ninja_deps file')
    args = parser.parse_args()

    if DEBUG_ALLOC:
        tracemalloc.start(25)
        tc_start = tracemalloc.take_snapshot()

    # Parse ninja file
    manifest = Parser().parse(args.input_file, args.encoding, args.ninja_deps)

    if DEBUG_ALLOC:
        tc_end = tracemalloc.take_snapshot()

    for rule in manifest.rules:
        print('rule', rule.name)

    for build in manifest.builds:
        print('build')
        for path in build.explicit_outs:
            print('  explicit_out:', path)
        for path in build.implicit_outs:
            print('  implicit_out:', path)
        for path in build.explicit_ins:
            print('  explicit_in:', path)
        for path in build.implicit_ins:
            print('  implicit_in:', path)
        for path in build.prerequisites:
            print('  prerequisites:', path)
        for path in build.depfile_implicit_ins:
            print('  depfile_implicit_in:', path)

    for pool in manifest.pools:
        print('pool', pool.name)

    for default in manifest.defaults:
        print('default')
        for path in default.outs:
            print('  out:', path)

    if DEBUG_ALLOC:
        top_stats = tc_end.compare_to(tc_start, 'traceback')
        with open('tracemalloc.log', 'w') as fp:
            for s in top_stats:
                print('', file=fp)
                print('========================================', file=fp)
                print(s, file=fp)
                for line in s.traceback.format():
                    print(line, file=fp)

def print_malloc_context(**kwargs):
    """
    :param \**kwargs: see print_malloc_snapshot
    """
    if tracemalloc is None:
        logger.error('tracemalloc required')
        return
    tracemalloc.start()
    yield
    snapshot = tracemalloc.take_snapshot()
    print_malloc_snapshot(snapshot, **kwargs)

def start():
    """ Starts application memory profiling """
    global tracemalloc

    logging.debug("Starting memory profiling")

    import tracemalloc

    with _lock:
        if is_running():
            raise RuntimeError('Memory profiler is already running')
        tracemalloc.start(_FRAMES)

 def measure_memory_diff(self, func):
     import tracemalloc
     tracemalloc.start()
     try:
         before = tracemalloc.take_snapshot()
         # Keep the result and only delete it after taking a snapshot
         res = func()
         after = tracemalloc.take_snapshot()
         del res
         return after.compare_to(before, 'lineno')
     finally:
         tracemalloc.stop()

def start_tracemalloc_dump():
    """
    If the environment variable W3AF_PYTRACEMALLOC is set to 1, then we start
    the thread that will dump the memory usage data which can be retrieved
    using tracemalloc module.

    :return: None
    """
    # save 25 frames
    tracemalloc.start(25)

    dump_data_every_thread(dump_tracemalloc, DELAY_MINUTES, SAVE_TRACEMALLOC_PTR)

    def test_track_already_tracked(self):
        nframe = 5
        tracemalloc.start(nframe)

        # track a first time
        self.track()

        # calling _PyTraceMalloc_Track() must remove the old trace and add
        # a new trace with the new traceback
        frames = self.track(nframe=nframe)
        self.assertEqual(self.get_traceback(),
                         tracemalloc.Traceback(frames))

 def exec_with_profiler(filename, profiler, backend):
     choose_backend(backend)
     if _backend == 'tracemalloc' and has_tracemalloc:
         tracemalloc.start()
     builtins.__dict__['profile'] = profiler
     # shadow the profile decorator defined above
     ns = dict(_CLEAN_GLOBALS, profile=profiler)
     try:
         with open(filename) as f:
             exec(compile(f.read(), filename, 'exec'), ns, ns)
     finally:
         if has_tracemalloc and tracemalloc.is_tracing():
             tracemalloc.stop()

def exec_with_profiler(filename, profiler, backend, passed_args=[]):
    from runpy import run_module
    builtins.__dict__['profile'] = profiler
    ns = dict(_CLEAN_GLOBALS, profile=profiler)
    _backend = choose_backend(backend)
    sys.argv = [filename] + passed_args
    try:
        if _backend == 'tracemalloc' and has_tracemalloc:
            tracemalloc.start()
        with open(filename) as f:
            exec(compile(f.read(), filename, 'exec'), ns, ns)
    finally:
        if has_tracemalloc and tracemalloc.is_tracing():
            tracemalloc.stop()

def memprofile():
    import resource
    import tracemalloc

    tracemalloc.start()

    ast = parse_file('/tmp/197.c')

    print('Memory usage: %s (kb)' %
            resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

    snapshot = tracemalloc.take_snapshot()
    print("[ tracemalloc stats ]")
    for stat in snapshot.statistics('lineno')[:20]:
        print(stat)

    def start(self):
        if self.profiling:
            return

        self.profiling = True

        now = datetime.now()
        trace_file = '{:%Y%m%d_%H%M}_trace.pickle'.format(now)
        trace_path = os.path.join(self.datadir, trace_file)
        self.trace_stream = open(trace_path, 'w')
        tracemalloc.start(15)

        # Take snapshots at slower pace because the size of the samples is not
        # negligible, the de/serialization is slow and uses lots of memory.
        self.timer = Timer(self.trace, interval=MINUTE * 5)