def get_next_treeherder_job(self):
        logger = utils.getLogger()
        conn = self._conn()

        job_cursor = self._execute_sql(
            conn,
            'select id,attempts,last_attempt,machine,project,job_collection from treeherder')

        job_row = job_cursor.fetchone()
        job_cursor.close()
        if not job_row:
            self._close_connection(conn)
            return None

        job = {'id': job_row[0],
               'attempts': job_row[1],
               'last_attempt': job_row[2],
               'machine': job_row[3],
               'project': job_row[4],
               'job_collection': json.loads(job_row[5])}
        job['attempts'] += 1
        job['last_attempt'] = datetime.datetime.utcnow().isoformat()
        self._execute_sql(
            conn,
            'update treeherder set attempts=?, last_attempt=? where id=?',
            values=(job['attempts'], job['last_attempt'],
                    job['id']))

        logger.debug('jobs.get_next_treeherder_job: %s', job)
        self._commit_connection(conn)
        self._close_connection(conn)
        return job

    def __init__(self, circle, fcp, total_chunks, totalsize=0,signature=False):
        BaseTask.__init__(self, circle)
        self.circle = circle
        self.fcp = fcp
        self.totalsize = totalsize
        self.signature = signature
        if self.signature:
            self.bfsign = BFsignature(total_chunks)


        # failed
        self.failed = {}

        self.failcnt = 0

        # debug
        self.d = {"rank": "rank %s" % circle.rank}
        self.logger = utils.getLogger(__name__)

        # reduce
        self.vsize = 0

        assert len(circle.workq) == 0

        if self.circle.rank == 0:
            print("\nChecksum verification ...")

def initAPIExport (apiName, psName, apiType, newAPIFlag):
	""" 
	Purpose: Method fo initializing the API export from Akana CM and PS (Production & Sandbox) export from Akana PM
	Input: Name of the API and Physical Service, the type of API (INTERNAL/EXTERNAL) and whether this is the first time API is being exported (Y/N)
	Returns: Dictionary object comprising of the relative path of the TAR file for API and PS	
	"""
	global mylogger
	mylogger = utils.getLogger ("APA-API-EXPORT")

	api_tar_path = ""
	if (apiName != ""):
		api_tar_path  = processCMExport (apiName, apiType, newAPIFlag)	
	
	ps_tar_path = ""
	pssb_tar_path = ""

	if (psName != ""):
		# Export Prod PS
		ps_tar_path  = processPMExport (psName, apiType, newAPIFlag)
		# Export Sandbox PS
		pssb_tar_path  = processPMExport (psName+"sb", apiType, newAPIFlag)
	
	tarFileDict = { "api_tar_path" : api_tar_path, "ps_tar_path" : ps_tar_path , "pssb_tar_path" : pssb_tar_path }

	return tarFileDict

def hook_tk_errors():
    """TKinter catches and swallows callback errors.

     we need to hook into that to log those seperately.
    """
    import traceback
    main_logger = utils.getLogger()

    def tk_error(exc_type, exc_value, exc_tb):
        """Log TK errors."""
        # The exception is caught inside the TK code.
        # We don't care about that, so try and move the traceback up
        # one level.
        if exc_tb.tb_next:
            exc_tb = exc_tb.tb_next
        main_logger.error(
            'TKinter callback exception occurred:\n{}',
            ''.join(
                traceback.format_exception(
                    exc_type,
                    exc_value,
                    exc_tb,
                )
            ),
        )
        # Don't shutdown - most of the time, the TK errors won't
        # shutdown the application.

    TK_ROOT.report_callback_exception = tk_error

    def post_request(self, machine, project, job_collection, attempts, last_attempt):
        logger = utils.getLogger()
        logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s',
                     job_collection.__dict__, attempts, last_attempt)

        try:
            self.client.post_collection(project, job_collection)
            return True
        except Exception, e:
            logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s',
                             attempts, last_attempt)
            if attempts > 1 and self.mailer:
                if hasattr(e, 'response') and e.response:
                    response_json = json.dumps(e.response.json(),
                                               indent=2, sort_keys=True)
                else:
                    response_json = None
                request_len = len(job_collection.to_json())
                self.mailer.send(
                    '%s attempt %d Error submitting request to Treeherder' %
                    (utils.host(), attempts),
                    'Phone: %s\n'
                    'Exception: %s\n'
                    'Last attempt: %s\n'
                    'Request length: %d\n'
                    'Response: %s\n' % (
                        machine,
                        e,
                        last_attempt,
                        request_len,
                        response_json))

def initSwaggerCreation(tarURL):
    """
	Purpose: Initiate the process of the TAR file containing the Swagger specifications for an API. The TAR file is downloaded from a Nexus URL and stored in 'source' directory.
	Since the Swagger files are embedded in a TAR-EAR-WAR structure the contents need to be extracted first and then modified
	The target directory is the one in which the modified swagger files will be created
	Input: URL of the TAR file containing the Swagger files
	Returns: List containing relative path of the modified Swagger zip files
	"""
    global mylogger
    mylogger = utils.getLogger("APA-SWG-PROC")
    try:
        mylogger.info("************************************")
        mylogger.info("BEGIN -- initSwaggerCreation()")

        if tarURL == "" or tarURL == None:
            raise Exception("Swagger TAR file name not provided")

        swaggerDir = swgConfig.get("SWG", "swaggerDir")
        tmpSwaggerLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "tmpSwaggerLocation"))
        swaggerSourceLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "swaggerSourceLocation"))

        urllib.request.urlretrieve(tarURL, os.path.join(swaggerSourceLocation, "tarname.tar"))

        sourcePathDict = extractSwaggerFromSource(swaggerSourceLocation, "tarname.tar")
        swaggerZipFileList = processSwagger(tmpSwaggerLocation, sourcePathDict)

        mylogger.info("END -- initSwaggerCreation()")
        mylogger.info("************************************")
        return swaggerZipFileList
    except:
        mylogger.error(traceback.format_exc())
        sys.exit()

 def serve_forever(self):
     logger = utils.getLogger()
     while not self.shutdown_requested:
         wait_seconds = 1    # avoid busy loop
         job = self.jobs.get_next_treeherder_job()
         if job:
             tjc = TreeherderJobCollection()
             for data in job['job_collection']:
                 tj = TreeherderJob(data)
                 tjc.add(tj)
             if self.post_request(job['machine'], job['project'], tjc,
                                  job['attempts'], job['last_attempt']):
                 self.jobs.treeherder_job_completed(job['id'])
                 wait_seconds = 0
             else:
                 attempts = int(job['attempts'])
                 wait_seconds = min(self.retry_wait * attempts, 3600)
                 logger.debug('AutophoneTreeherder waiting for %d seconds after '
                              'failed attempt %d',
                              wait_seconds, attempts)
         if wait_seconds > 0:
             for i in range(wait_seconds):
                 if self.shutdown_requested:
                     break
                 time.sleep(1)

def modify(request):
    if request.method == 'GET':
        id=request.GET.get('id')
        m=getMonitorById(id)
        if m:
            #sb=str(m.smsBegin)
            #se=str(m.smsEnd)
            return render_to_response('modify.html',{'monitor':m})
        else:
            return HttpResponse('id is not exisit')
    else:
        m=modifyMonitor(
            oldName=request.POST.get('oldName'),
            log=getLogger(),
            name=request.POST.get('name'),
            interval=request.POST.get('interval'),
            mailInterval=request.POST.get('mailInterval'),
            smsInterval=request.POST.get('smsInterval'),
            smsBegin=request.POST.get('smsBegin'),
            smsEnd=request.POST.get('smsEnd'),
            retryTimes=request.POST.get('retryTimes'),
            retryInterval=request.POST.get('retryInterval'),
            describe=request.POST.get('describe'),
            url=request.POST.get('url'),
            timeout=request.POST.get('timeout'),
            contacts=request.POST.get('contacts'),
            statusCode=request.POST.get('statusCode'),
            status=False,
        )
        if m:
            return HttpResponseRedirect('/index/')
        else:
            return HttpResponse('modify failed')

 def treeherder_job_completed(self, th_id):
     logger = utils.getLogger()
     logger.debug('jobs.treeherder_job_completed: %s', th_id)
     conn = self._conn()
     self._execute_sql(conn, 'delete from treeherder where id=?', values=(th_id,))
     self._commit_connection(conn)
     self._close_connection(conn)

 def test_completed(self, test_guid):
     logger = utils.getLogger()
     logger.debug('jobs.test_completed: %s', test_guid)
     conn = self._conn()
     self._execute_sql(conn, 'delete from tests where guid=?', values=(test_guid,))
     self._commit_connection(conn)
     self._close_connection(conn)

 def stop(self):
     """Stops the pulse monitor listen thread."""
     logger = utils.getLogger()
     logger.debug('AutophonePulseMonitor stopping')
     self._stopping.set()
     self.listen_thread.join()
     logger.debug('AutophonePulseMonitor stopped')

    def get_treeherder_privatebuild_info(self, project, job):
        logger = utils.getLogger()
        url = '%s/api/jobdetail/?repository=%s&job_id=%s' % (
            self.treeherder_url, project, job['id'])
        data = utils.get_remote_json(url)
        logger.debug("get_treeherder_privatebuild_info: data: %s", data)

        privatebuild_keys = set(['build_url', 'config_file', 'chunk',
                                 'builder_type'])
        info = {}
        for detail in data['results']:
            if detail['title'] in privatebuild_keys:
                # the build_url property is too long to fit into "value"
                if detail.get('url'):
                    value = detail['url']
                else:
                    value = detail['value']
                info[detail['title']] = value

        # If we're missing a privatebuild key for some reason
        # return None to avoid errors.
        missing_keys = privatebuild_keys - set(info.keys())
        if missing_keys:
            logger.debug("get_treeherder_privatebuild_info: %s "
                         "missing keys: %s "
                         "job: %s", url, missing_keys, job)
            return None

        return info

def add(request):
    if request.method == 'GET':
        return render_to_response('add.html')
    else:
        m=createMonitor(
            log=getLogger(),
            name=request.POST.get('name'),
            interval=request.POST.get('interval'),
            mailInterval=request.POST.get('mailInterval'),
            smsInterval=request.POST.get('smsInterval'),
            smsBegin=request.POST.get('smsBegin'),
            smsEnd=request.POST.get('smsEnd'),
            retryTimes=request.POST.get('retryTimes'),
            retryInterval=request.POST.get('retryInterval'),
            describe=request.POST.get('describe'),
            url=request.POST.get('url'),
            timeout=request.POST.get('timeout'),
            contacts=request.POST.get('contacts'),
            statusCode=request.POST.get('statusCode'),
            status=False,
        )
        if m:
            return HttpResponseRedirect('/index/')
        else:
            return HttpResponse('create failed')

    def upload(self, path, destination):
        try:
            logger = utils.getLogger()
            key = self.bucket.get_key(destination)
            if not key:
                logger.debug('Creating key: %s', destination)
                key = self.bucket.new_key(destination)

            ext = os.path.splitext(path)[-1]
            if ext == '.log' or ext == '.txt':
                key.set_metadata('Content-Type', 'text/plain')

            with tempfile.NamedTemporaryFile('w+b', suffix=ext) as tf:
                logger.debug('Compressing: %s', path)
                with gzip.GzipFile(path, 'wb', fileobj=tf) as gz:
                    with open(path, 'rb') as f:
                        gz.writelines(f)
                tf.flush()
                tf.seek(0)
                key.set_metadata('Content-Encoding', 'gzip')
                logger.debug('Setting key contents from: %s', tf.name)
                key.set_contents_from_file(tf)

            url = key.generate_url(expires_in=0,
                                   query_auth=False)
        except boto.exception.S3ResponseError, e:
            logger.exception(str(e))
            raise S3Error('%s' % e)

    def __init__(self, circle, fcp, totalsize=0):
        BaseTask.__init__(self, circle)
        self.circle = circle
        self.fcp = fcp
        self.totalsize = totalsize

        # cache
        self.fd_cache = LRU(512)

        # failed
        self.failed = {}

        self.failcnt = 0

        # debug
        self.d = {"rank": "rank %s" % circle.rank}
        self.logger = utils.getLogger(__name__)

        # reduce
        self.vsize = 0

        assert len(circle.workq) == 0

        if self.circle.rank == 0:
            print("\nChecksum verification ...")

    def __init__(self, worker_subprocess, options, jobs, s3_bucket=None,
                 mailer=None):
        assert options, "options is required."

        logger = utils.getLogger()

        self.options = options
        self.jobs = jobs
        self.s3_bucket = s3_bucket
        self.mailer = mailer
        self.worker = worker_subprocess
        self.shutdown_requested = False
        logger.debug('AutophoneTreeherder')

        self.url = self.options.treeherder_url
        if not self.url:
            logger.debug('AutophoneTreeherder: no treeherder url')
            return

        self.client_id = self.options.treeherder_client_id
        self.secret = self.options.treeherder_secret
        self.retry_wait = self.options.treeherder_retry_wait

        self.client = TreeherderClient(server_url=self.url,
                                       client_id=self.client_id,
                                       secret=self.secret)

        logger.debug('AutophoneTreeherder: %s', self)

 def job_completed(self, job_id):
     logger = utils.getLogger()
     logger.debug('jobs.job_completed: %s', job_id)
     conn = self._conn()
     self._execute_sql(conn, 'delete from tests where jobid=?', values=(job_id,))
     self._execute_sql(conn, 'delete from jobs where id=?', values=(job_id,))
     self._commit_connection(conn)
     self._close_connection(conn)

 def handle(self):
     try:
         data = ''
         while True:
             data = self.connection.recv(4)
             if len(data) < 4:
                 break
             pickle_len = struct.unpack('>L', data)[0]
             data = self.connection.recv(pickle_len)
             while len(data) < pickle_len:
                 data = data + self.connection.recv(pickle_len - len(data))
             record = logging.makeLogRecord(pickle.loads(data))
             logger = utils.getLogger(record.name)
             logger.handle(record)
     except:
         logger = utils.getLogger()
         logger.exception("Error receiving log record: data: %s", data)

 def delete_anr_traces(self, root=True):
     """Empty ANR traces.txt file."""
     try:
         self.adb.rm(TRACES, root=root)
         self.adb.shell_output('echo > %s' % TRACES, root=root)
         self.adb.chmod(TRACES, mask='666', root=root)
     except ADBError, e:
         logger = utils.getLogger()
         logger.warning("Could not initialize ANR traces %s, %s", TRACES, e)

    def __init__(self, cfgfile, subject_prefix=''):
        self.cfgfile = cfgfile
        self.subject_prefix = subject_prefix
        self.from_address = None
        self.mail_dest = None
        self.mail_username = None
        self.mail_password = None
        self.mail_server = None
        self.mail_port = None
        self.mail_ssl = None

        logger = utils.getLogger()

        cfg = ConfigParser.ConfigParser()
        if not cfg.read(self.cfgfile):
            logger.info('No email configuration file found. No emails will be sent.')
            return

        try:
            self.from_address = cfg.get('report', 'from')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            logger.error('No "from" option defined in "report" section '
                         'of file "%s".\n', self.cfgfile)
            return

        try:
            self.mail_dest = [x.strip() for x in cfg.get('email', 'dest').split(',')]
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            logger.error('No "dest" option defined in "email" section '
                         'of file "%s".\n', self.cfgfile)
            return

        try:
            self.mail_username = cfg.get('email', 'username')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.mail_username = None

        try:
            self.mail_password = cfg.get('email', 'password')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.mail_password = None

        try:
            self.mail_server = cfg.get('email', 'server')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.mail_server = 'smtp.mozilla.org'

        try:
            self.mail_port = cfg.getint('email', 'port')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.mail_port = 25

        try:
            self.mail_ssl = cfg.getboolean('email', 'ssl')
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.mail_ssl = False