def seconds_until_run_time(self):
     run_time = self.run_time
     if run_time.tzinfo:
         now = timeutils.current_time(tz=run_time.tzinfo)
     else:
         now = timeutils.current_time()
     return max(0, timeutils.delta_total_seconds(run_time - now))

 def test_sleep_time(self):
     assert_equal(mcp.sleep_time(timeutils.current_time()), 0)
     assert_equal(mcp.sleep_time(timeutils.current_time() - datetime.timedelta(seconds=5)), 0)
     
     seconds = 5
     time = mcp.sleep_time(timeutils.current_time() + datetime.timedelta(seconds=seconds)) 
     assert equals_with_delta(time, seconds, .01)

    def test_success_but_recent_failure(self):
        job_run = turtle.Turtle()
        job_run.is_success = True
        job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30)
        self.job.runs.append(job_run)

        bad_job_run = turtle.Turtle()
        bad_job_run.is_success = False
        bad_job_run.end_time = timeutils.current_time()
        self.job.runs.append(bad_job_run)

        assert self.resource.is_ready

    def start(self):
        log.info("Starting action job %s", self.id)
        self.start_time = timeutils.current_time()
        self.end_time = None

        for action in self.action_runs:
            action.attempt_start()

    def start(self):
        if not self.machine.check('start'):
            raise InvalidStartStateError(self.state)

        log.info("Starting action run %s", self.id)
        self.start_time = timeutils.current_time()
        self.end_time = None
        self.machine.transition('start')
        assert self.state == self.STATE_STARTING, self.state
        self._open_output_file()

        if not self.is_valid_command:
            log.error("Command for action run %s is invalid: %r",
                      self.id, self.action.command)
            self.fail(-1)
            return

        # And now we try to actually start some work....
        self.action_command = ActionCommand(self.id,
                                            self.command,
                                            stdout=self.stdout_file,
                                            stderr=self.stderr_file)
        self.action_command.machine.listen(True, self._handle_action_command)
        try:
            df = self.node.run(self.action_command)
            df.addErrback(self._handle_errback)
        except node.Error, e:
            log.warning("Failed to start %s: %r", self.id, e)

    def render_GET(self, request):
        run_output = []
        state = job_run_state(self._run)

        for action_run in self._run.runs:
            action_state = job_run_state(action_run)

            last_time = action_run.end_time if action_run.end_time else timeutils.current_time()
            duration = str(last_time - action_run.start_time) if action_run.start_time else ""

            run_output.append(
                {
                    "id": action_run.id,
                    "name": action_run.action.name,
                    "run_time": action_run.run_time and str(action_run.run_time),
                    "start_time": action_run.start_time and str(action_run.start_time),
                    "end_time": action_run.end_time and str(action_run.end_time),
                    "exit_status": action_run.exit_status,
                    "duration": duration,
                    "state": action_state,
                }
            )

        output = {"runs": run_output, "id": self._run.id, "state": state, "node": self._run.node.hostname}

        return respond(request, output)

    def start(self):
        log.info("Starting action job %s", self.job.name)
        self.start_time = timeutils.current_time()
        self.end_time = None

        for r in self.runs:
            r.attempt_start()

    def _check_job_runs(self):
        if timeutils.current_time() < self.next_check_time:
            return

        min_success_time = None
        if self.last_succeed_interval:
            min_success_time = timeutils.current_time() - self.last_succeed_interval

        for run in reversed(self.job.runs):
            if run.is_success and (min_success_time is None or run.end_time >= min_success_time):
                self._is_ready = True
                break
        else:
            self._is_ready = False

        self.last_check = timeutils.current_time()

    def test_success(self):
        job_run = turtle.Turtle()
        job_run.is_success = True
        job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30)
        self.job.runs.append(job_run)

        assert self.resource.is_ready

    def render_GET(self, request):
        run_output = []
        state = job_run_state(self._run)
        
        for action_run in self._run.action_runs:
            action_state = job_run_state(action_run)
            
            last_time = action_run.end_time if action_run.end_time else timeutils.current_time()
            duration = str(last_time - action_run.start_time) if action_run.start_time else ""
           
            run_output.append({
                'id': action_run.id,
                'name': action_run.action.name,
                'run_time': action_run.run_time and str(action_run.run_time),
                'start_time': action_run.start_time and str(action_run.start_time),
                'end_time': action_run.end_time and str(action_run.end_time),
                'exit_status': action_run.exit_status,
                'duration': duration,
                'state': action_state,
                'command': action_run.command,
            })

        output = {
            'runs': run_output, 
            'id': self._run.id, 
            'state': state,
            'node': self._run.node.hostname,
        }
        
        return respond(request, output)

    def render_POST(self, request):
        cmd = request.args['command'][0]

        log.info("Handling '%s' request for job run %s", cmd, self._job.name)

        if cmd == 'enable':
            self._master_control.enable_job(self._job)
            return respond(request, {'result': "Job %s is enabled" % self._job.name})

        if cmd == 'disable':
            self._master_control.disable_job(self._job)
            return respond(request, {'result': "Job %s is disabled" % self._job.name})

        if cmd == 'start':
            if 'run_time' in request.args:
                run_time_str = request.args['run_time'][0]
                run_time = datetime.datetime.strptime(run_time_str, "%Y-%m-%d %H:%M:%S")
            else:
                run_time = timeutils.current_time()
            
            runs = self._job.manual_start(run_time=run_time)
            return respond(request, {'result': "New Job Runs %s created" % [r.id for r in runs]})

        log.warning("Unknown request job command %s", request.args['command'])
        return respond(request, None, code=http.NOT_IMPLEMENTED)

 def manual_start(self, run_time=None):
     """Trigger a job run manually (instead of from the scheduler)."""
     run_time = run_time or timeutils.current_time()
     manual_runs = list(self.job.build_new_runs(run_time, manual=True))
     for r in manual_runs:
         r.start()
     return manual_runs

    def test_success_too_old(self):
        job_run = turtle.Turtle()
        job_run.is_success = True
        job_run.end_time = timeutils.current_time() - datetime.timedelta(days=1)
        self.job.runs.append(job_run)

        assert not self.resource.is_ready

 def _done(self, target, exit_status=0):
     log.info("Action run %s completed with %s and exit status %r",
         self.id, target, exit_status)
     if self.machine.check(target):
         self.exit_status = exit_status
         self.end_time = timeutils.current_time()
         return self.machine.transition(target)

    def next_runs(self, job):
        # Find the next time to run
        if job.runs:
            start_time = job.runs[0].run_time
        else:
            start_time = timeutils.current_time()
            if self.time_zone:
                try:
                    start_time = self.time_zone.localize(start_time,
                                                         is_dst=None)
                except AmbiguousTimeError:
                    # We are in the infamous 1 AM block which happens twice on
                    # fall-back. Pretend like it's the first time, every time.
                    start_time = self.time_zone.localize(start_time,
                                                         is_dst=True)
                except NonExistentTimeError:
                    # We are in the infamous 2:xx AM block which does not
                    # exist. Pretend like it's the later time, every time.
                    start_time = self.time_zone.localize(start_time,
                                                         is_dst=True)

        run_time = self.time_spec.GetMatch(start_time)

        job_runs = job.build_runs()
        for job_run in job_runs:
            job_run.set_run_time(run_time)

        return job_runs

    def next_runs(self, job):
        # Find the next time to run
        if job.runs:
            next_day = job.runs[0].run_time + datetime.timedelta(days=self.wait_days[job.runs[0].run_time.weekday()])
        else:
            next_day = timeutils.current_time() + datetime.timedelta(self.wait_days[timeutils.current_time().weekday()])

        run_time = next_day.replace(
                            hour=self.start_time.hour, 
                            minute=self.start_time.minute, 
                            second=self.start_time.second)

        job_runs = job.build_runs()
        for job_run in job_runs:
            job_run.set_run_time(run_time)
 
        return job_runs

    def test_set_run_time(self):
        jr = self.job.next_runs()[0]
        time = timeutils.current_time()
        jr.set_run_time(time)

        assert_equal(jr.run_time, time)
        assert_equal(jr.action_runs[0].run_time, time)
        assert_equal(jr.action_runs[1].run_time, time)

 def fail(self, exit_status=0):
     """Mark the run as having failed, providing an exit status"""
     log.info("Action run %s failed with exit status %r",
              self.id, exit_status)
     if self.machine.transition('fail'):
         self.exit_status = exit_status
         self.end_time = timeutils.current_time()
         return True

    def fail(self, exit_status):
        """Mark the run as having failed, providing an exit status"""
        log.info("Action run %s failed with exit status %r", self.id, exit_status)

        self.state = ACTION_RUN_FAILED
        self.exit_status = exit_status
        self.end_time = timeutils.current_time()
        self.complete_callback()
        self.state_callback()

 def seconds_until_run_time(self):
     run_time = self.run_time
     tz = self.job.scheduler.time_zone
     now = timeutils.current_time()
     if tz is not None:
         now = tz.localize(now)
     sleep = run_time - now
     seconds = (sleep.days * SECS_PER_DAY + sleep.seconds +
                sleep.microseconds * MICRO_SEC)
     return max(0, seconds)