def run_guarded(self, context):
        period = context.period

        if config.log_level == "processes":
            print()

        try:
            for k, v in self.subprocesses:
                if config.log_level == "processes":
                    print("    *", end=' ')
                    if k is not None:
                        print(k, end=' ')
                    utils.timed(v.run_guarded, context)
                else:
                    v.run_guarded(context)
                    #            print "done."
                context.simulation.start_console(context)
        finally:
            if config.autodump is not None:
                self._autodump(context)

            if config.autodiff is not None:
                self._autodiff(period)

            if self.purge:
                self.entity.purge_locals()

def upload_box(mail, api, cmax=None):
  result, data = mail.search(None, "ALL")
  if result == 'OK':
    ids = data[0].split() # Ids is a space separated string.
    ids.reverse()
    if cmax is None:
      cmax = len(ids)
    else:
      cmax = min(cmax, len(ids))
    i = 0
    tfetch = 0
    tupload = 0
    print('0.0 %', end='')
    for id in ids[:cmax]:
      # Fetch raw message.
      (result, data), elapsed = timed(mail.fetch, id, "(RFC822)") # fetch the email body (RFC822) for the given ID.
      tfetch += elapsed
      # Encode.
      raw = data[0][1] # Raw mail, in bytes.
      data, cte = _decode(raw)
      # Upload into API.
      try:
        result, elapsed = timed(api.message_insert, 'media', data, cte)
        tupload += elapsed
      except ApiError as err:
        print('\nFailed to upload one message: {}\n'.format(err))
      # Progress.
      i = i+1
      percent = i * 100 / cmax
      print('\r\033[0K{0:.2f} %'.format(percent), end='')
    print('\r\033[0KDone.\n-- fetch time: {}\n-- upload time {}'.format(tfetch, tupload))
  else:
    print('Could not access the mail box: {}'.format(result))

        def simulate_period(period_idx, period, processes, entities,
                            init=False):
            print("\nperiod", period)
            if init:
                for entity in entities:
                    print("  * %s: %d individuals" % (entity.name,
                                                      len(entity.array)))
            else:
                print("- loading input data")
                for entity in entities:
                    print("  *", entity.name, "...", end=' ')
                    timed(entity.load_period_data, period)
                    print("    -> %d individuals" % len(entity.array))
            for entity in entities:
                entity.array_period = period
                entity.array['period'] = period

            if processes:
                # build context for this period:
                const_dict = {'__simulation__': self,
                              'period': period,
                              'nan': float('nan'),
                              '__globals__': globals_data}

                num_processes = len(processes)
                for p_num, process_def in enumerate(processes, start=1):
                    process, periodicity = process_def

                    print("- %d/%d" % (p_num, num_processes), process.name,
                          end=' ')
                    print("...", end=' ')
                    if period_idx % periodicity == 0:
                        elapsed, _ = gettime(process.run_guarded, self,
                                             const_dict)
                    else:
                        elapsed = 0
                        print("skipped (periodicity)")

                    process_time[process.name] += elapsed
                    if config.show_timings:
                        print("done (%s elapsed)." % time2str(elapsed))
                    else:
                        print("done.")
                    self.start_console(process.entity, period,
                                       globals_data)

            print("- storing period data")
            for entity in entities:
                print("  *", entity.name, "...", end=' ')
                timed(entity.store_period_data, period)
                print("    -> %d individuals" % len(entity.array))
#            print " - compressing period data"
#            for entity in entities:
#                print "  *", entity.name, "...",
#                for level in range(1, 10, 2):
#                    print "   %d:" % level,
#                    timed(entity.compress_period_data, level)
            period_objects[period] = sum(len(entity.array)
                                         for entity in entities)

        def simulate_period(period_idx, period, processes, entities, init=False):
            print "\nperiod", period
            if init:
                for entity in entities:
                    print "  * %s: %d individuals" % (entity.name, len(entity.array))
            else:
                print "- loading input data"
                for entity in entities:
                    print "  *", entity.name, "...",
                    timed(entity.load_period_data, period)
                    print "    -> %d individuals" % len(entity.array)
            for entity in entities:
                entity.array_period = period
                entity.array["period"] = period

            if processes:
                # build context for this period:
                const_dict = {"period": period, "nan": float("nan"), "__globals__": globals_data}

                num_processes = len(processes)
                for p_num, process_def in enumerate(processes, start=1):
                    process, periodicity = process_def

                    print "- %d/%d" % (p_num, num_processes), process.name,
                    # TODO: provide a custom __str__ method for Process &
                    # Assignment instead
                    if hasattr(process, "predictor") and process.predictor and process.predictor != process.name:
                        print "(%s)" % process.predictor,
                    print "...",
                    if period_idx % periodicity == 0:
                        elapsed, _ = gettime(process.run_guarded, self, const_dict)
                    else:
                        elapsed = 0
                        print "skipped (periodicity)"

                    process_time[process.name] += elapsed
                    if config.show_timings:
                        print "done (%s elapsed)." % time2str(elapsed)
                    else:
                        print "done."
                    self.start_console(process.entity, period, globals_data)

            print "- storing period data"
            for entity in entities:
                print "  *", entity.name, "...",
                timed(entity.store_period_data, period)
                print "    -> %d individuals" % len(entity.array)
            #            print " - compressing period data"
            #            for entity in entities:
            #                print "  *", entity.name, "...",
            #                for level in range(1, 10, 2):
            #                    print "   %d:" % level,
            #                    timed(entity.compress_period_data, level)
            period_objects[period] = sum(len(entity.array) for entity in entities)

 def run_guarded(self, simulation, const_dict):
     print
     for k, v in self.subprocesses:
         print "    *",
         if k is not None:
             print k,
         utils.timed(v.run_guarded, simulation, const_dict)
         #            print "done."
         simulation.start_console(v.entity, const_dict["period"], const_dict["__globals__"])
     # purge all local variables
     temp_vars = self.entity.temp_variables
     all_vars = self.entity.variables
     local_vars = set(temp_vars.keys()) - set(all_vars.keys())
     for var in local_vars:
         del temp_vars[var]

    def aggregate(self, report_id):
        logbook.info("Get customer usage aggregation for {}", report_id)
        customer = Customer.get_by_id(report_id.customer_id)
        if not customer:
            raise Exception("Customer %s not found" % report_id.customer_id)

        with timed("get_usage simple"):
            aggregated_usage = ServiceUsage.get_usage(customer, report_id.start, report_id.end)

        tariffs = {}
        services = set()
        for usage in aggregated_usage:
            service_id, tariff_id, cost, usage_volume = usage
            services.add(service_id)
            if not tariff_id:
                logbook.error("ServiceUsage {} is not completed. Tariff is not filled", usage)
                continue
            tariff = Tariff.get_by_id(tariff_id)
            tariff_report = tariffs.get(tariff_id)
            if tariff_report is None:
                tariff_report = self.tariff_report_type(tariff, customer)
                tariffs[tariff_id] = tariff_report

            tariff_report.add_usage(usage)

        total = Counter()
        for tariff_id, tariff in tariffs.items():
            total_tariff, currency = tariff.aggregate()
            total[currency] += total_tariff

        for t, value in total.items():
            total[t] = decimal_to_string(value)

        logbook.info("Aggregated {} for {}. Services: {}", total, customer, services)
        return self.prepare_result(list(tariffs.values()), total, customer, report_id.start, report_id.end)

    def run_guarded(self, simulation, const_dict):
        period = const_dict['period']

        print()
        for k, v in self.subprocesses:
            print("    *", end=' ')
            if k is not None:
                print(k, end=' ')
            utils.timed(v.run_guarded, simulation, const_dict)
#            print "done."
            simulation.start_console(v.entity, period,
                                     const_dict['__globals__'])
        if config.autodump is not None:
            self._autodump(period)

        if config.autodiff is not None:
            self._autodiff(period)

        if self.purge:
            self.entity.purge_locals()

    def run_guarded(self, simulation, const_dict):
        global max_vars
        
        periods = const_dict['periods']
        idx = const_dict['period_idx']
        period =  periods[idx]
        
        print()
        for k, v in self.subprocesses:
#             print("    *", end=' ')
            if k is not None:
                print(k, end=' ')
            utils.timed(v.run_guarded, simulation, const_dict)
#            print "done."
            simulation.start_console(v.entity, period,
                                     const_dict['__globals__'])
        if config.autodump is not None:
            self._autodump(period)

        if config.autodiff is not None:
            self._autodiff(period)

        # purge all local variables
        temp_vars = self.entity.temp_variables
        all_vars = self.entity.variables
        local_var_names = set(temp_vars.keys()) - set(all_vars.keys())
        num_locals = len(local_var_names)
        if config.debug and num_locals:
            local_vars = [v for k, v in temp_vars.iteritems()
                          if k in local_var_names and
                             isinstance(v, np.ndarray)]
            max_vars = max(max_vars, num_locals)
            temp_mem = sum(v.nbytes for v in local_vars)
            avgsize = sum(v.dtype.itemsize for v in local_vars) / num_locals
            print(("purging {} variables (max {}), will free {} of memory "
                  "(avg field size: {} b)".format(num_locals, max_vars,
                                                  utils.size2str(temp_mem),
                                                  avgsize)))

        for var in local_var_names:
            del temp_vars[var]

def report_file_generate(self, report_id):
    from report import Report
    from memdb.report_cache import ReportCache, ReportTask

    report_cache = ReportCache()

    aggregated = report_cache.get_report_aggregated(report_id)
    if not aggregated:
        aggregated = get_aggregation(report_id)
        aggregated = ReportCache.unpack_aggregated(ReportCache.pack_aggregated(aggregated))
    report_generator = Report.get_report(report_id.report_type)
    with timed("rendering for %s" % report_id):
        data = report_generator.render(aggregated, report_id)
    report_cache.set_report(report_id, data, report_generator.report_cache_time)
    ReportTask().remove(report_id)

def user_tiles(adapter, count=5000):

    query = timed(adapter.query_tile_time_percent)

    time_sum = 0
    with open('./tiles.txt') as f:
        queries = (tuple(int(x) for x in line.split('/')[1:5]) for line in f)
        for zoom, resolution, x, y in itertools.islice(queries, count):
        
            time_v, count_v = query(x, y, zoom, resolution, 0.0, 1.0)
            time_sum += time_v.microseconds

            time.sleep(DELAY)
        
        average = time_sum / count / 1.e6
        print 'User queries: average time {}'.format(average)

    def filter_and_group(usage):
        usage_by_resource = defaultdict(list)
        with timed("filter and group by resource"):
            trust_sources = set(conf.fitter.trust_sources)
            for u in usage:
                # the user can make their own samples, including those
                # that would collide with what we care about for
                # billing.
                # if we have a list of trust sources configured, then
                # discard everything not matching.
                if trust_sources and u.source not in trust_sources:
                    logbook.warning('ignoring untrusted usage sample from source `{}`', u['source'])
                    continue

                resource_id = u.resource_id
                usage_by_resource[resource_id].append(u)
        return usage_by_resource

    def get_tenant_usage(self, tenant_id, meter_name, start, end, limit=None):
        """ Queries ceilometer for all the entries in a given range,
           for a given meter, from this tenant."""

        query = [self.filter('timestamp', 'ge', start), self.filter('timestamp', 'lt', end)]

        if tenant_id:
            query.append(self.filter('project_id', 'eq', tenant_id))

        if meter_name:
            query.append(self.filter('meter', 'eq', meter_name))

        with timed('fetch global usage for meter %s' % meter_name):
            result = openstack.client_ceilometer.new_samples.list(q=query, limit=limit)
            log.debug("Get usage for tenant: {} and meter_name {} ({} - {}). Number records: {}",
                      tenant_id, meter_name, start, end, len(result))
            return result

def region_time(adapter, latlon_mag=160, latlon_step=160, time_steps=5, zoom=4):
    
    lat0s = lat1s = lon0s = lon1s = [float(x) / 180. for x in range(-latlon_mag, latlon_mag + latlon_step, latlon_step)]
    starts = ends = [float(x) / time_steps for x in range(time_steps)]
    
    query = timed(adapter.query_region_latlon_time_percent)
    
    count = 0
    time_sum = 0.
    for lat0, lat1, lon0, lon1, start, end in itertools.product(lat0s, lat1s, lon0s, lon1s, starts, ends):
        if lat0 >= lat1 or lon0 >= lon1 or start >= end: continue
                    
        time_v, count_v = query(lat0, lon0, lat1, lon1, zoom, start, end)
        count += 1
        time_sum += time_v.microseconds
    
        time.sleep(DELAY)

    print 'For regions, average time {}'.format(time_sum / count / 1.e6)

def tile_time_resolution(adapter, time_steps=5, zoom=4, tile_samples=20, resolution=8):

    random.seed(1)
    all_coords = list(itertools.product(range(0, 2**zoom), range(0, 2**zoom)))
    coords = random.sample(all_coords, tile_samples)

    starts = ends = [float(x) / time_steps for x in range(time_steps)]

    query = timed(adapter.query_tile_time_percent)

    count = 0
    time_sum = 0
    for start, end, (x, y) in itertools.product(starts, ends, coords):
        if start >= end: continue
    
        time_v, count_v = query(x, y, zoom, resolution, start, end)
        count += 1
        time_sum += time_v.microseconds

        time.sleep(DELAY)
    
    average = time_sum / count / 1.e6
    print 'At resolution {} average time {}'.format(resolution, average)

import utils

from contextlib import closing

from django.db import connection
from django.utils import timezone


def sql_batch_insert(n_records):
    sql = 'INSERT INTO app_testmodel (field_1, field_2, field_3) VALUES {}'.format(
        ', '.join(['(%s, %s, %s)'] * n_records),
    )
    params = []
    for i in xrange(0, n_records):
        params.extend([i, str(i), timezone.now()])

    with closing(connection.cursor()) as cursor:
        cursor.execute(sql, params)


if __name__ == '__main__':
    utils.timed(sql_batch_insert)

    # copy globals
    if copy_globals:
        # noinspection PyProtectedMember
        input_file.root.globals._f_copy(output_file.root, recursive=True)

    output_entities = output_file.create_group("/", "entities", "Entities")
    for table in input_file.iterNodes(input_file.root.entities):
        # noinspection PyProtectedMember
        print(table._v_name, "...")
        copy_table(table, output_entities, condition=condition)

    input_file.close()
    output_file.close()


if __name__ == '__main__':
    import sys
    import platform

    print("LIAM HDF5 filter %s using Python %s (%s)\n" %
          (__version__, platform.python_version(), platform.architecture()[0]))

    args = dict(enumerate(sys.argv))
    if len(args) < 4:
        print("""Usage: {} inputpath outputpath condition [copy_globals]
where condition is an expression
      copy_globals is True (default)|False""".format(args[0]))
        sys.exit()

    timed(filter_h5, args[1], args[2], args[3], eval(args.get(4, 'True')))

        def simulate_period(period_idx, period, periods, processes, entities,
                            init=False):
            period_start_time = time.time()

            # set current period
            eval_ctx.period = period

            if config.log_level in ("procedures", "processes"):
                print()
            print("period", period,
                  end=" " if config.log_level == "periods" else "\n")
            if init and config.log_level in ("procedures", "processes"):
                for entity in entities:
                    print("  * %s: %d individuals" % (entity.name,
                                                      len(entity.array)))
            else:
                if config.log_level in ("procedures", "processes"):
                    print("- loading input data")
                    for entity in entities:
                        print("  *", entity.name, "...", end=' ')
                        timed(entity.load_period_data, period)
                        print("    -> %d individuals" % len(entity.array))
                else:
                    for entity in entities:
                        entity.load_period_data(period)
            for entity in entities:
                entity.array_period = period
                entity.array['period'] = period

            if processes:
                # build context for this period:
                const_dict = {'period_idx': period_idx + 1,
                              'periods': periods,
                              'periodicity': time_period[self.time_scale] * (1 - 2 * (self.retro)),
                              'longitudinal': self.longitudinal,
                              'format_date': self.time_scale,
                              'pension': None,
                              '__simulation__': self,
                              'period': period,
                              'nan': float('nan'),
                              '__globals__': globals_data}
                assert(periods[period_idx + 1] == period)

                num_processes = len(processes)
                for p_num, process_def in enumerate(processes, start=1):

                    process, periodicity, start = process_def
                    if config.log_level in ("procedures", "processes"):
                        print("- %d/%d" % (p_num, num_processes), process.name,
                              end=' ')
                        print("...", end=' ')
                    # TDOD: change that
                    if isinstance(periodicity, int):
                        if period_idx % periodicity == 0:
                            elapsed, _ = gettime(process.run_guarded, self,
                                                 const_dict)
                        else:
                            elapsed = 0
                            print("skipped (periodicity)")
                    else:
                        assert periodicity in time_period
                        periodicity_process = time_period[periodicity]
                        periodicity_simul = time_period[self.time_scale]
                        month_idx = period % 100
                        # first condition, to run a process with start == 12
                        # each year even if year are yyyy01
                        # modify start if periodicity_simul is not month
                        start = int(start / periodicity_simul - 0.01) * periodicity_simul + 1

                        if (periodicity_process <= periodicity_simul and self.time_scale != 'year0') or (
                                month_idx % periodicity_process == start % periodicity_process):

                            const_dict['periodicity'] = periodicity_process * (1 - 2 * (self.retro))
                            elapsed, _ = gettime(process.run_guarded, self, const_dict)
                        else:
                            elapsed = 0

                        if config.log_level in ("procedures", "processes"):
                            print("skipped (periodicity)")

                    process_time[process.name] += elapsed
                    if config.log_level in ("procedures", "processes"):
                        if config.show_timings:
                            print("done (%s elapsed)." % time2str(elapsed))
                        else:
                            print("done.")
                    self.start_console(eval_ctx)

            # update longitudinal
            person = [x for x in entities if x.name == 'person'][0]
            # maybe we have a get_entity or anything more nice than that #TODO: check
            id = person.array.columns['id']

            for varname in ['sali', 'workstate']:
                var = person.array.columns[varname]
                if init:
                    fpath = self.data_source.input_path
                    input_file = HDFStore(fpath, mode="r")
                    if 'longitudinal' in input_file.root:
                        input_longitudinal = input_file.root.longitudinal
#.........这里部分代码省略.........

    def run(self, run_console=False):
        start_time = time.time()

        h5in, h5out, globals_data = timed(self.data_source.run,
                                          self.globals_def,
                                          entity_registry,
                                          self.init_period)

        if config.autodump or config.autodiff:
            if config.autodump:
                fname, _ = config.autodump
                mode = 'w'
            else:  # config.autodiff
                fname, _ = config.autodiff
                mode = 'r'
            fpath = os.path.join(config.output_directory, fname)
            h5_autodump = tables.open_file(fpath, mode=mode)
            config.autodump_file = h5_autodump
        else:
            h5_autodump = None

#        input_dataset = self.data_source.run(self.globals_def,
#                                             entity_registry)
#        output_dataset = self.data_sink.prepare(self.globals_def,
#                                                entity_registry)
#        output_dataset.copy(input_dataset, self.init_period - 1)
#        for entity in input_dataset:
#            indexed_array = buildArrayForPeriod(entity)

        # tell numpy we do not want warnings for x/0 and 0/0
        np.seterr(divide='ignore', invalid='ignore')

        process_time = defaultdict(float)
        period_objects = {}
        eval_ctx = EvaluationContext(self, self.entities_map, globals_data)

        def simulate_period(period_idx, period, periods, processes, entities,
                            init=False):
            period_start_time = time.time()

            # set current period
            eval_ctx.period = period

            if config.log_level in ("procedures", "processes"):
                print()
            print("period", period,
                  end=" " if config.log_level == "periods" else "\n")
            if init and config.log_level in ("procedures", "processes"):
                for entity in entities:
                    print("  * %s: %d individuals" % (entity.name,
                                                      len(entity.array)))
            else:
                if config.log_level in ("procedures", "processes"):
                    print("- loading input data")
                    for entity in entities:
                        print("  *", entity.name, "...", end=' ')
                        timed(entity.load_period_data, period)
                        print("    -> %d individuals" % len(entity.array))
                else:
                    for entity in entities:
                        entity.load_period_data(period)
            for entity in entities:
                entity.array_period = period
                entity.array['period'] = period

            if processes:
                # build context for this period:
                const_dict = {'period_idx': period_idx + 1,
                              'periods': periods,
                              'periodicity': time_period[self.time_scale] * (1 - 2 * (self.retro)),
                              'longitudinal': self.longitudinal,
                              'format_date': self.time_scale,
                              'pension': None,
                              '__simulation__': self,
                              'period': period,
                              'nan': float('nan'),
                              '__globals__': globals_data}
                assert(periods[period_idx + 1] == period)

                num_processes = len(processes)
                for p_num, process_def in enumerate(processes, start=1):

                    process, periodicity, start = process_def
                    if config.log_level in ("procedures", "processes"):
                        print("- %d/%d" % (p_num, num_processes), process.name,
                              end=' ')
                        print("...", end=' ')
                    # TDOD: change that
                    if isinstance(periodicity, int):
                        if period_idx % periodicity == 0:
                            elapsed, _ = gettime(process.run_guarded, self,
                                                 const_dict)
                        else:
                            elapsed = 0
                            print("skipped (periodicity)")
                    else:
                        assert periodicity in time_period
                        periodicity_process = time_period[periodicity]
                        periodicity_simul = time_period[self.time_scale]
                        month_idx = period % 100
#.........这里部分代码省略.........

 def load(self):
     return timed(self.data_source.load, self.globals_def, self.entities_map)

import utils

from django.utils import timezone

from app import models


def orm_bulk_create(n_records):
    instances = [
        models.TestModel(
            field_1=i,
            field_2=str(i),
            field_3=timezone.now(),
        )
        for i in xrange(0, n_records)
    ]

    models.TestModel.objects.bulk_create(instances)


if __name__ == '__main__':
    utils.timed(orm_bulk_create)