def run(self, force_current_time="no", force_last_time="no"):
     """
     Runs all registered jobs that currently need to perform
     updates.  If the optional *force_current_time* or
     *force_last_time* arguments are included, jobs will be run as if
     working at those times.  (This is useful for testing or force
     loading old data.)
     """
     jobs = list(self._jobs)
     random.shuffle(jobs)
     for job_name in jobs:
         if force_current_time == "no" or force_current_time == None:
             start_time = make_datetime(datetime.utcnow())
         else:
             start_time = make_datetime(force_current_time)
         (period, func) = self._jobs[job_name]
         current_time = bin_datetime(period, start_time)
         if force_last_time == "no" or force_last_time == None:
             last_time = self._last_run_times.get(job_name, None)
         else:
             last_time = make_datetime(force_last_time)
         if last_time is None or last_time < current_time:
             try:
                 log.debug("%s %s %s %s %s" % (job_name, period, last_time, current_time, func))
                 func(last_time, current_time)
                 self._last_run_times[job_name] = current_time
             except:
                 # We don't fail on errors.  The func has to do something
                 # if it wants to be special.
                 log.exception("Job manager: error running %s" % job_name)
                 pass

def nice_month_ticks(lo, hi, ticks=5, inside=False):
    lo_year = lo.year
    lo_month = lo.month - 1
    hi_year = hi.year
    hi_month = hi.month - 1
    if hi - make_datetime(datetime(hi_year, hi.month, 1)):
        if hi_month == 11:
            hi_month = 0
            hi_year += 1
        else:
            hi_month += 1
    delta_year = hi_year - lo_year
    if delta_year * 12 + hi_month - lo_month < (ticks - 1):
        raise ValueError()
    t_min, t_max, t_iter = nice_ticks(lo_month, delta_year * 12 + hi_month,
                                      ticks, inside,
                                      intervals=int_12_intervals, base=12.0)
    t_min_year = int(lo_year + t_min / 12)
    t_min_month = int(t_min % 12) + 1
    t_max_year = int(lo_year + t_max / 12)
    t_max_month = int(t_max % 12) + 1
    month_min = make_datetime(datetime(t_min_year, t_min_month, 1))
    month_max = make_datetime(datetime(t_max_year, t_max_month, 1))
    def m_iter():
        for t in t_iter:
            year = int(lo_year + t / 12)
            month = int(t % 12) + 1
            yield make_datetime(datetime(year, month, 1))
    return (month_min, month_max, m_iter())

 def test_nice_time_ticks_16(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2013-01-01'))
     self.assertEqual(a, make_datetime('2011-01-01'))
     self.assertEqual(b, make_datetime('2013-01-01'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01', '2011-07-01', '2012-01-01',
                                 '2012-07-01', '2013-01-01']])

 def test_nice_time_ticks_11(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2011-01-14'))
     self.assertEqual(a, make_datetime('2011-01-01'))
     self.assertEqual(b, make_datetime('2011-01-16'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01', '2011-01-04', '2011-01-07',
                                 '2011-01-10', '2011-01-13', '2011-01-16']])

 def test_nice_time_ticks_17(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2525-01-01'))
     self.assertEqual(a, make_datetime('2000-01-01'))
     self.assertEqual(b, make_datetime('2600-01-01'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2000-01-01', '2100-01-01', '2200-01-01',
                                 '2300-01-01', '2400-01-01', '2500-01-01',
                                 '2600-01-01']])

 def test_nice_time_ticks_12(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2011-02-24'))
     self.assertEqual(a, make_datetime('2011-01-01'))
     self.assertEqual(b, make_datetime('2011-03-02'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01', '2011-01-11', '2011-01-21',
                                 '2011-01-31', '2011-02-10', '2011-02-20',
                                 '2011-03-02']])

 def test_nice_time_ticks_9(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2011-01-03'))
     self.assertEqual(a, make_datetime('2011-01-01'))
     self.assertEqual(b, make_datetime('2011-01-03'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01T00', '2011-01-01T12',
                                 '2011-01-02T00', '2011-01-02T12',
                                 '2011-01-03T00']])

 def test_nice_time_ticks_15(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2011-12-01'), ticks=12)
     self.assertEqual(a, make_datetime('2011-01-01'))
     self.assertEqual(b, make_datetime('2011-12-01'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01', '2011-02-01', '2011-03-01',
                                 '2011-04-01', '2011-05-01', '2011-06-01',
                                 '2011-07-01', '2011-08-01', '2011-09-01',
                                 '2011-10-01', '2011-11-01', '2011-12-01']])

 def test_nice_time_ticks_6(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
                                 make_datetime('2011-01-01T01:00:00.000'))
     self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
     self.assertEqual(b, make_datetime('2011-01-01T01:00:00.000'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01T00:00:00.000',
                                 '2011-01-01T00:15:00.000',
                                 '2011-01-01T00:30:00.000',
                                 '2011-01-01T00:45:00.000',
                                 '2011-01-01T01:00:00.000']])

 def test_nice_time_ticks_1(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01'),
                                 make_datetime('2011-01-01'))
     self.assertEqual(a, make_datetime('2010-12-31T23:59:59.500'))
     self.assertEqual(b, make_datetime('2011-01-01T00:00:00.500'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2010-12-31T23:59:59.500',
                                 '2010-12-31T23:59:59.750',
                                 '2011-01-01T00:00:00.000',
                                 '2011-01-01T00:00:00.250',
                                 '2011-01-01T00:00:00.500']])

 def test_nice_time_ticks_7(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
                                 make_datetime('2011-01-01T14:00:00.000'))
     self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
     self.assertEqual(b, make_datetime('2011-01-01T15:00:00.000'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01T00:00:00.000',
                                 '2011-01-01T03:00:00.000',
                                 '2011-01-01T06:00:00.000',
                                 '2011-01-01T09:00:00.000',
                                 '2011-01-01T12:00:00.000',
                                 '2011-01-01T15:00:00.000']])

 def test_nice_time_ticks_5(self):
     (a, b, i) = nice_time_ticks(make_datetime('2011-01-01T00:00:00.000'),
                                 make_datetime('2011-01-01T00:10:00.000'))
     self.assertEqual(a, make_datetime('2011-01-01T00:00:00.000'))
     self.assertEqual(b, make_datetime('2011-01-01T00:10:00.000'))
     self.assertEqual(list(i), [make_datetime(x) for x in
                                ['2011-01-01T00:00:00.000',
                                 '2011-01-01T00:02:00.000',
                                 '2011-01-01T00:04:00.000',
                                 '2011-01-01T00:06:00.000',
                                 '2011-01-01T00:08:00.000',
                                 '2011-01-01T00:10:00.000']])

def nice_year_ticks(lo, hi, ticks=5, inside=False):
    lo_year = lo.year
    hi_year = hi.year
    if hi - make_datetime(datetime(hi_year, 1, 1)):
        hi_year += 1
    if hi_year - lo_year < (ticks - 1):
        raise ValueError()
    t_min, t_max, t_iter = nice_ticks(lo_year, hi_year, ticks, inside,
                                      intervals=int_intervals)
    year_min = make_datetime(datetime(int(t_min), 1, 1))
    year_max = make_datetime(datetime(int(t_max), 1, 1))
    def y_iter():
        for t in t_iter:
            yield make_datetime(datetime(int(t), 1, 1))
    return (year_min, year_max, y_iter())

def parse_param_date_value(param, value):
    try:
        value = make_datetime(value)
    except ValueError:
        error = ParamError(param, value, "not a valid date/time")
        raise error
    return value

def parse_param_date_value(param, value, is_default):
    if is_default:
        default_msg = " in script default"
    else:
        default_msg = ""
    try:
        value = make_datetime(value)
    except ValueError:
        error = ParamError(param, value, "not a valid date/time" + default_msg)
        raise error
    return value

 def _make_from_meta(cls, **kwargs):
     mdata = kwargs.setdefault('metadata', {})
     version = int(mdata['golem_version'])
     if version > GOLEM_VERSION:
         error = GolemScriptError(
             "Unknown golem version %s" % repr(version))
         raise error
     kwargs['_version'] = version
     if 'golem_span' in mdata:
         kwargs['span'] = \
             times.make_timedelta(mdata.pop('golem_span'))
     if 'golem_interval' in mdata:
         kwargs['interval'] = \
             times.make_timedelta(mdata.pop('golem_interval'))
     if 'golem_lag' in mdata:
         kwargs['lag'] = \
             times.make_timedelta(mdata.pop('golem_lag'))
     if 'golem_epoch' in mdata:
         kwargs['epoch'] = \
             times.make_datetime(mdata.pop('golem_epoch'))
     if 'golem_realtime' in mdata:
         kwargs['realtime'] = bool(mdata.pop('golem_realtime'))
     if 'golem_name' in mdata:
         kwargs['name'] = mdata.pop('golem_name')
     if 'golem_suite_name' in mdata:
         kwargs['suite_name'] = mdata.pop('golem_suite_name')
     if 'golem_inputs' in mdata:
         inputs = []
         for meta, spec in mdata.pop('golem_inputs'):
             inputs.append((cls._make_from_meta(**meta), spec))
         kwargs['golem_inputs'] = inputs
     if 'golem_tags' in mdata:
         kwargs['tags'] = mdata.pop('golem_tags')
     if 'golem_loops' in mdata:
         kwargs['loops'] = mdata.pop('golem_loops')
     if 'golem_flow_maps' in mdata:
         kwargs['flow_maps'] = mdata.pop('golem_flow_maps')
     if 'golem_arg_tags' in mdata:
         kwargs['arg_tags'] = mdata.pop('golem_arg_tags')
     if 'golem_repository' in mdata:
         kwargs['repository'] = mdata.pop('golem_repository')
     if 'golem_output_templates' in mdata:
         kwargs['output_templates'] = mdata.pop('golem_output_templates')
     if 'golem_input_templates' in mdata:
         kwargs['input_templates'] = mdata.pop('golem_input_templates')
     if 'golem_query_templates' in mdata:
         kwargs['query_templates'] = mdata.pop('golem_query_templates')
     if 'output_groups' in mdata:
         kwargs['output_groups'] = mdata.pop('output_groups')
     if 'input_groups' in mdata:
         kwargs['input_groups'] = mdata.pop('input_groups')
     return cls(**kwargs)

 def store(self, item):
     """
     Stores the provided data structure into the associated file,
     along with a timestamp representing the current time.
     """
     try:
         if DEBUG:
             print >> sys.stderr, "save to sig file %s" % self.file
         fh = open(self.file, 'wb')
         pickle.dump((item, make_datetime(datetime.now())), fh)
         fh.close()
     except pickle.PicklingError, e:
         msg = "problem storing %s : %s" % (self.file, e)
         raise LedgerStoreError, msg

def date_from_file(file):
    """
    Attempt to extract dates from a filename. The filename can be a
    full pathname or relative path. Dates are presumed to exist
    somewhere in the pathname.  See :func:`split_on_date` for more
    detail on how dates are parsed from filenames.
    """
    (dir, chop) = split_on_date(file)
    ymd = {}
    for (i, field) in ( (1, 'year'), (3, 'month'),  (5, 'day'),
                        (7, 'hour'), (9, 'minute'), (11, 'second') ):
        if field in ('day', 'month'):
            ymd[field] = chop[i] or 1
        else:
            ymd[field] = chop[i] or 0
    try:
        return make_datetime(datetime(**ymd), utc_only=True)
    except ValueError, e:
        msg = str(e)
        msg += "\nproblem extracting date from %s" % file
        raise DateFileParseError, msg

    def __init__(self, span=None, interval=None,
                       name=None, suite_name=None, epoch=None,
                       lag=None, realtime=False, repository=None,
                       tags=None, arg_tags=None, loops=None,
                       flow_maps=None, golem_inputs=None,
                       output_templates=None, input_templates=None,
                       query_templates=None, input_groups=None,
                       output_groups=None, _version=None, **kwargs):
        """
        Creates a new :class:`Golem` object from the various data that
        make up a golem script definition. This should never need to be
        used by anything outside of the :mod:`golem.model` module. See
        instead :func:`golem.model.parse_golem_metadata` for a way to
        retrieve golem script information from serialized metadata or
        existing scripts.
        """

        Script.__init__(self, **kwargs)

        self._version = _version

        self.name = name
        """
        The short name of this golem script, if any.
        """
        if self.name is None:
            try:
                self.name = os.path.basename(main.__file__)
                self.name.replace('.py', '')
            except AttributeError:
                self.name = "python"

        self.suite = suite_name
        """
        The short name of this golem script's suite, if any.
        """

        self._epoch = None
        """
        The epoch which all time bins are relative to. Defaults to
        midnight of the first Monday after the standard unix epoch.
        """
        if epoch:
                self._epoch = times.make_datetime(epoch)

        self.interval = \
            times.make_timedelta(interval or self.default_interval)
        """
        A :class:`timedelta` representing the process interval
        of this :class:`Golem` instance.
        """

        self.span = times.make_timedelta(span or self.interval)
        """
        A :class:`timedelta` representing the data window of
        this :class:`Golem` instance.
        """

        self.repository = repository
        """
        The default path, if any, for the output results data
        repository for this :class:`Golem` instance.
        """

        self.realtime = realtime or False
        """
        A boolean value indicating whether results for this
        :class:`Golem` script can be generated in real time
        or not. If true, forces a *lag* of 0.
        """

        self._lag = None
        if lag and not self.realtime:
            self._lag = times.make_timedelta(lag)

        if not isinstance(tags, dict):
            tags = dict(tags or ())
        self.tags = tags

        if not isinstance(arg_tags, dict):
            arg_tags = dict(arg_tags or ())
        self.arg_tags = arg_tags

        self.flow_maps = tuple(flow_maps or ())
        for x in self.flow_maps:
            name, params = x
            if isinstance(x, tuple) and isinstance(params, dict):
                continue
            maps = []
            for name, params in self.flow_maps:
                maps.append((name, dict(params)))
            self.flow_maps = tuple(flow_maps)
            break

        self.loops = tuple(loops or ())
        for item in self.loops:
            name, x = item
            vals, groups, group_name, sep = x
            if all(isinstance(y, tuple) for y in (item, x, vals)) and \
                    not isinstance(vals, basestring) and \
#.........这里部分代码省略.........

 def test_make_datetime_iso_usec_tzminute(self):
     "make_datetime('2010-02-03T04:05:06.007008+09:10')"
     self.assertEqual(
         self.dt_20100203T040506_007008_0910,
         make_datetime('2010-02-03T04:05:06.007008+09:10'))