def ver_from_rpm_repo(self, repo, branch=None):
     out = {}
     rpm_repo_url_template = repo["rpm_repo_url"]
     if rpm_repo_url_template:
         rpm_repo_url_template = rpm_repo_url_template.strip()
         if branch:
             rpm_repo_url_template = rpm_repo_url_template % branch
         for release in ["5", "6", "7"]:
             rpm_repo_url = rpm_repo_url_template.replace("$releasever", release)
             rpm_repo_url = rpm_repo_url.replace("$basearch", "x86_64")
             url = os.path.join(rpm_repo_url, "repodata/primary.xml.gz")
             try:
                 r = requests.get(url)
                 r.raise_for_status()
                 assert r.text, "Empty primary.xml file"
                 s = StringIO.StringIO(r.content)
                 f = gzip.GzipFile(fileobj=s, mode="r")
                 f.seek(0)
                 xml = minidom.parse(f)
                 try:
                     out[rpm_repo_url_template] = self.rpm_pattern_1.findall(xml.toxml())[0].strip()
                 except:
                     out[rpm_repo_url_template] = self.rpm_pattern_2.findall(xml.toxml())[0].strip()
             except (requests.exceptions.HTTPError, requests.exceptions.InvalidSchema):
                 msg = "RPM repository {0} failed, file not found: {1}"
                 msg = msg.format(repo["rpm_repo_url"], url)
                 LOG.warning(msg)
         return out

    def _is_server_for_update(self, server, status):
        repo_url = status["repo_url"]
        devel_branch = server.get("user-data.scm_branch", None)
        ver_info = self.get_szr_ver_from_repo(devel_branch=devel_branch)

        try:
            szr_ver_repo = ver_info[repo_url]
        except KeyError:
            pkg_type = helper.pkg_type_by_name(status["dist"].split()[0])
            szr_ver_repo = ver_info[status["repository"]][pkg_type]

        if parse_version(server["scalarizr.version"]) >= parse_version(szr_ver_repo):
            return False
        if "in-progress" in status["state"]:
            # skip in-progress server
            return False
        if status["executed_at"]:
            last_update_dt = datetime.datetime.strptime(status["executed_at"], "%a %d %b %Y %H:%M:%S %Z")
            last_update_dt = last_update_dt.replace(minute=0, second=0, microsecond=0)
            utcnow_dt = datetime.datetime.utcnow()
            utcnow_dt = utcnow_dt.replace(minute=0, second=0, microsecond=0)
            if last_update_dt == utcnow_dt and status["state"] == "error":
                # skip failed server
                LOG.debug("Skip server: {0}, reason: server in error state".format(server["server_id"]))
                return False
        return True

    def _is_server_for_update(self, server, status):
        repo_url = status['repo_url']
        devel_branch = server.get('user-data.scm_branch', None)
        ver_info = self.get_szr_ver_from_repo(devel_branch=devel_branch)

        try:
            szr_ver_repo = ver_info[repo_url]
        except KeyError:
            pkg_type = helper.pkg_type_by_name(status['dist'].split()[0])
            szr_ver_repo = ver_info[status['repository']][pkg_type]

        if parse_version(server['scalarizr.version']) >= parse_version(szr_ver_repo):
            return False
        if 'in-progress' in status['state']:
            # skip in-progress server
            return False
        if status['executed_at']:
            last_update_dt = datetime.datetime.strptime(
                status['executed_at'], '%a %d %b %Y %H:%M:%S %Z')
            last_update_dt = last_update_dt.replace(minute=0, second=0, microsecond=0)
            utcnow_dt = datetime.datetime.utcnow()
            utcnow_dt = utcnow_dt.replace(minute=0, second=0, microsecond=0)
            if last_update_dt == utcnow_dt and status['state'] == 'error':
                # skip failed server
                LOG.debug(
                    'Skip server: {0}, reason: server in error state'.format(server['server_id']))
                return False
        return True

 def ver_from_rpm_repo(self, repo, branch=None):
     out = {}
     rpm_repo_url_template = repo['rpm_repo_url']
     if rpm_repo_url_template:
         rpm_repo_url_template = rpm_repo_url_template.strip()
         if branch:
             rpm_repo_url_template = rpm_repo_url_template % branch
         for release in ['5', '6']:
             rpm_repo_url = rpm_repo_url_template.replace('$releasever', release)
             rpm_repo_url = rpm_repo_url.replace('$basearch', 'x86_64')
             url = os.path.join(rpm_repo_url, 'repodata/primary.xml.gz')
             try:
                 r = requests.get(url)
                 r.raise_for_status()
             except (requests.exceptions.HTTPError, requests.exceptions.InvalidSchema):
                 msg = 'RPM repository {0} failed, file: {1} not found'.format(repo['rpm_repo_url'], url)
                 LOG.warning(msg)
                 return out
             s = StringIO.StringIO(r.content)
             f = gzip.GzipFile(fileobj=s, mode='r')
             f.seek(0)
             xml = minidom.parse(f)
             try:
                 out[rpm_repo_url_template] = self.rpm_pattern_1.findall(xml.toxml())[0].strip()
             except:
                 out[rpm_repo_url_template] = self.rpm_pattern_2.findall(xml.toxml())[0].strip()
         return out

    def download_aws_billing_file(self, cred, bucket_name, date=None):
        if date is None:
            date = datetime.datetime.utcnow().date()

        conn = get_s3_conn(cred)
        bucket = conn.get_bucket(bucket_name)
        account_id = cryptotool.decrypt_scalr(app.crypto_key, cred['account_id'])
        file_name = get_aws_csv_file_name(account_id, date)
        key = bucket.get_key(file_name)

        if not key:
            msg = "AWS detailed billing CSV file {0} wasn't found in bucket {1}"
            msg = msg.format(file_name, bucket_name)
            if datetime.datetime.utcnow().day == 1:
                LOG.warning(msg)
                return None
            else:
                raise Exception(msg)

        last_modified_dt = datetime.datetime.strptime(key.last_modified, self.last_modified_format)
        update_interval = self.config['interval']
        utcnow = datetime.datetime.utcnow()
        delta = datetime.timedelta(seconds=update_interval)
        condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta
        condition2 = ((utcnow - last_modified_dt).seconds / 3600) % 8 == 0
        if condition1 or condition2:
            local_file_path = os.path.join(self.tmp_dir, file_name)
            LOG.debug('Downloading {0}'.format(file_name))
            key.get_contents_to_filename(local_file_path)
            return local_file_path
        else:
            return None

    def do_iteration(self):
        while len(self._processing_messages) > self.max_processing_messages:
            time.sleep(1)
        messages = self.get_messages()
        if not messages:
            time.sleep(self.nothing_todo_sleep)
            return

        servers = self.get_servers(messages)
        servers_map = dict((server['server_id'], server) for server in servers)

        for message in messages:
            try:
                if message['messageid'] in self._processing_messages:
                    continue
                self._processing_messages.add(message['messageid'])

                if message['server_id'] not in servers_map:
                    msg = (
                        "Server '{server_id}' doesn't exist or not in right status, set message "
                        "status to 3"
                    ).format(server_id=message['server_id'])
                    LOG.warning(msg)
                    message['status'] = 3
                    self._pool.wait()
                    self._pool.apply_async(self.update, (message,))
                else:
                    server = servers_map[message['server_id']]
                    self._pool.wait()
                    self._pool.apply_async(self.process_message, (message, server))
            except:
                msg = "Unable to process message: {message_id}, reason: {error}"
                msg = msg.format(message_id=message['messageid'], error=helper.exc_info())
                LOG.warning(msg)

 def _serve_forever(self):
     LOG.debug('Starting plotter')
     try:
         cherrypy.quickstart(self, '/', {'/': {}})
     except:
         LOG.error(helper.exc_info())
         thread.interrupt_main()

 def __call__(self):
     self.change_permissions()
     while True:
         try:
             self.iteration_timestamp = time.time()
             self.before_iteration()
             g = self._do_iteration()
             try:
                 g.get(timeout=self.iteration_timeout)
             except:
                 self.on_iteration_error()
                 raise
             finally:
                 if not g.ready():
                     g.kill()
                 self.after_iteration()
             iteration_time = time.time() - self.iteration_timestamp
             msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
             LOG.debug(msg)
         except:
             LOG.exception('Iteration failed')
             time.sleep(self.error_sleep)
         finally:
             if self.config['interval']:
                 next_iteration_time = self.iteration_timestamp + self.config['interval']
                 time.sleep(next_iteration_time - time.time())

def _get_szr_conn_info(server, port, instances_connection_policy):
    ip = {
        'public': server['remote_ip'],
        'local': server['local_ip'],
        'auto': server['remote_ip'] if server['remote_ip'] else server['local_ip'],
    }[instances_connection_policy]
    headers = {}
    if server['platform'] == 'ec2' and 'ec2.vpc.id' in server and 'router.vpc.ip' in server:
        if server['remote_ip']:
            ip = server['remote_ip']
        else:
            headers.update({
                'X-Receiver-Host': server['local_ip'],
                'X-Receiver-Port': port,
            })
            ip = server['router.vpc.ip']
            port = 80

    # Start - Added by Chen Leji
    if not server['remote_ip']:
        LOG.info("=============Apply FLOATINGIP_PROXY Patch=============")
        ip   = "localhost"
        proxy = floatingip_proxy.szrProxy(port, server['server_id'])
        port = proxy.get_proxy_port()

    return ip, port, headers

    def get_servers(self, limit=500):
        for servers in self._get_servers(limit=limit):

            prop = ['scalarizr.api_port', 'scalarizr.key']
            self._db.load_server_properties(servers, prop)

            for server in servers:
                if 'scalarizr.api_port' not in server:
                    server['scalarizr.api_port'] = 8010
                if 'scalarizr.key' not in server:
                    server['scalarizr.key'] = None

            self._db.load_vpc_settings(servers)

            out = []
            for server in servers:
                try:
                    if server['os_type'] == 'linux':
                        exclude = ['snum']
                    elif server['os_type'] == 'windows':
                        exclude = ['la', 'io', 'snum']
                    else:
                        msg = "Wrong os type for server: '%s'" % server['server_id']
                        raise Exception(msg)
                    metrics = [m for m in self.config['metrics'] if m not in exclude]
                    server['metrics'] = metrics
                    out.append(server)
                except:
                    LOG.error(helper.exc_info())
                    continue
            yield out

    def do_iteration(self):
        messages = self.get_messages()
        if not messages:
            raise exceptions.NothingToDoError()

        servers = self.get_servers(messages)
        servers_map = dict((server['server_id'], server) for server in servers)

        for message in messages:
            try:
                if message['server_id'] not in servers_map:
                    msg = (
                        "Server '{server_id}' doesn't exist or not in right status, set message "
                        "status to 3"
                    ).format(server_id=message['server_id'])
                    LOG.warning(msg)
                    message['status'] = 3
                    self._pool.wait()
                    self._pool.apply_async(self.update, (message,))
                else:
                    server = servers_map[message['server_id']]
                    self._pool.wait()
                    self._pool.apply_async(self.process_message, (message, server))
            except:
                msg = "Unable to process message: {message_id}, reason: {error}"
                msg = msg.format(message_id=message['messageid'], error=helper.exc_info())
                LOG.warning(msg)
        self._pool.join()

def get_net_stat(hsp, api_type='linux', timeout=5):
    net = hsp.sysinfo.net_stats(timeout=timeout)
    if api_type == 'linux':
        ret = {
            'in': float(net['eth0']['receive']['bytes']),
            'out': float(net['eth0']['transmit']['bytes']),
        }
    elif api_type == 'windows':
        for key in net:
            if re.match(r'^.*Ethernet Adapter.*$', key) \
                    or re.match(r'^.*AWS PV Network Device.*$', key):
                ret = {
                    'in': float(net[key]['receive']['bytes']),
                    'out': float(net[key]['transmit']['bytes']),
                }
                break
        else:
            msg = (
                "Can't find ['^.* Ethernet Adapter.*$', '^.*AWS PV Network Device.*$'] "
                "pattern in api response for endpoint: {0}, available: {1}, use {2}"
            ).format(hsp.endpoint, net.keys(), net.keys()[0])
            LOG.warning(msg)
            first_key = net.keys()[0]
            ret = {
                'in': float(net[first_key]['receive']['bytes']),
                'out': float(net[first_key]['transmit']['bytes']),
            }
    else:
        raise APIError("Unsupported API type '%s' for NET stat" % api_type)
    return ret

 def execute(self, query, retries=0, retry_timeout=10):
     while True:
         try:
             if self._autocommit or not self._connection:
                 self._local.connection = self._connection_pool.get(timeout=10)
             self._local.connection.autocommit(self._autocommit)
             self._local.cursor = self._connection.cursor()
             try:
                 start_time = time.time()
                 self._local.cursor.execute(query)
                 end_time = time.time()
                 if end_time - start_time > 1:
                     LOG.debug('Query too slow: %s\n%s...' %
                               (end_time - start_time, query[:150]))
                 results = self._local.cursor.fetchall()
                 if results is not None:
                     results = tuple(results)
                 return results
             finally:
                 if self._autocommit:
                     self._local.cursor.close()
                     self._connection_pool.put(self._local.connection)
                     self._local.connection = None
                     self._local.cursor = None
         except (pymysql.err.OperationalError, pymysql.err.InternalError, socket.timeout):
             if not retries:
                 raise
             retries -= 1
             time.sleep(retry_timeout)

def get_metrics(host, port, key, api_type, metrics, headers=None, timeout=5):
    assert host, 'host'
    assert port, 'port'
    assert key, 'key'
    assert api_type, 'api_type'
    assert metrics, 'metrics'

    data = dict()
    endpoint = 'http://%s:%s' % (host, port)
    security = rpc.Security(key)
    hsp = rpc.HttpServiceProxy(endpoint, security=security, headers=headers)
    getters = {
        'cpu': get_cpu_stat,
        'la': get_la_stat,
        'mem': get_mem_info,
        'net': get_net_stat,
        'io': get_io_stat,
    }
    for metric in metrics:
        try:
            data.update({metric: getters[metric](hsp, api_type, timeout=timeout)})
        except (urllib2.URLError, urllib2.HTTPError, socket.timeout):
            msg = "Endpoint: {endpoint}, headers: {headers}, metric: '{metric}', reason: {err}"
            msg = msg.format(
                endpoint=endpoint, headers=headers, metric=metric, err=helper.exc_info())
            raise Exception(msg)
        except:
            msg = "Endpoint: {endpoint}, headers: {headers}, metric '{metric}' failed, reason: {er}"
            msg = msg.format(
                endpoint=endpoint, headers=headers, metric=metric, err=helper.exc_info())
            LOG.warning(msg)
            continue
    return data

 def do_iteration(self):
     for envs in self.analytics.load_envs():
         try:
             self.analytics.load_env_credentials(envs)
             unique = {}
             for env in envs:
                 try:
                     credentials = self.analytics.get_credentials([env])
                     for cred in credentials:
                         if cred.platform == 'ec2' and env.get('ec2.detailed_billing.enabled', '0') == '1':
                             continue
                         unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred})
                         unique[cred.unique]['envs_ids'].append(env['id'])
                 except:
                     msg = 'Processing environment: {} failed'.format(env['id'])
                     LOG.exception(msg)
             for data in unique.values():
                 while len(self.pool) > self.config['pool_size'] * 5 / 10:
                     gevent.sleep(0.1)
                 self.pool.apply_async(process_credential,
                                       args=(data['cred'],),
                                       kwds={'envs_ids': data['envs_ids']})
                 gevent.sleep(0)  # force switch
         except:
             msg = 'Processing environments: {} failed'.format([env['id'] for env in envs])
             LOG.exception(msg)
     self.pool.join()

def handle_error(message=None, level='exception'):
    c, e, t = sys.exc_info()
    if message:
        message = message.rstrip().rstrip('.') + '. Reason: {}'.format(exc_info())
    else:
        message = exc_info()
    if isinstance(e, (
                      KeyboardInterrupt,
                      GeneratorExit,
                      greenlet_mod.GreenletExit,
                      gevent.Timeout,
                     )
                  ):
        LOG.debug(message)
        raise
    if isinstance(e, SystemExit) and sys.exc_info()[1].args[0] == 0:
        raise
    logging_map = {
        'debug': LOG.debug,
        'info': LOG.info,
        'warning': LOG.warning,
        'error': LOG.error,
        'critical': LOG.critical,
        'exception': LOG.exception,
    }
    if isinstance(e, pymysql.err.Error):
        logging_map[min(level, 'error')](message)
    else:
        logging_map[level](message)

 def __call__(self):
     self.change_permissions()
     while True:
         try:
             self.iteration_timestamp = time.time()
             self.before_iteration()
             g = self._do_iteration()
             try:
                 g.get(timeout=self.iteration_timeout)
             except:
                 self.on_iteration_error()
                 raise
             finally:
                 if not g.ready():
                     g.kill()
                 self.after_iteration()
         except:
             try:
                 helper.handle_error(message='Iteration failed')
             except (SystemExit, KeyboardInterrupt):
                 return
             except:
                 pass
             time.sleep(self.error_sleep)
         finally:
             iteration_time = time.time() - self.iteration_timestamp
             msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
             LOG.debug(msg)
             if self.config['interval']:
                 next_iteration_time = self.iteration_timestamp + self.config['interval']
                 sleep_time = next_iteration_time - time.time()
                 if sleep_time:
                     time.sleep(sleep_time)

    def _average(self, results, ra=None, fa=None, rs=None, fs=None):
        ra = ra or dict()
        fa = fa or dict()
        rs = rs or dict()
        fs = fs or dict()
        for result in results:
            try:
                r_key, f_key = self._get_rf_keys(result)
                if 'snum' in self.config['metrics']:
                    rs.setdefault(r_key, {'snum': {'s_running': 0}})
                    fs.setdefault(f_key, {'snum': {'s_running': 0}})
                    rs[r_key]['snum']['s_running'] += 1
                    fs[f_key]['snum']['s_running'] += 1
                if not result['data']:
                    continue
                for metrics_group_name, metrics_data in result['data'].iteritems():
                    if not metrics_data or metrics_group_name == 'io':
                        continue
                    for metric_name, value in metrics_data.iteritems():
                        try:
                            ra.setdefault(r_key, {})
                            ra[r_key].setdefault(metrics_group_name, {})
                            ra[r_key][metrics_group_name].setdefault(metric_name, Average())

                            fa.setdefault(f_key, {})
                            fa[f_key].setdefault(metrics_group_name, {})
                            fa[f_key][metrics_group_name].setdefault(metric_name, Average())

                            ra[r_key][metrics_group_name][metric_name] += value
                            fa[f_key][metrics_group_name][metric_name] += value
                        except:
                            LOG.error(helper.exc_info())
            except:
                LOG.error(helper.exc_info())
        return ra, fa, rs, fs

def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
    LOG.debug("Daemonize")

    # first fork
    pid = os.fork()
    if pid > 0:
        sys.exit(0)

    os.chdir('/')
    os.setsid()
    os.umask(0)

    # second fork
    pid = os.fork()
    if pid > 0:
        sys.exit(0)

    # redirect standard file descriptors
    sys.stdout.flush()
    sys.stderr.flush()
    si = file(stdin, 'r')
    so = file(stdout, "a+")
    se = file(stderr, "a+", 0)
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(so.fileno(), sys.stdout.fileno())
    os.dup2(se.fileno(), sys.stderr.fileno())

def create_pid_file(pid_file):
    pid = str(os.getpid())
    msg = "Creating pid file: %s" % pid_file
    LOG.debug(msg)
    if not os.path.exists(os.path.dirname(pid_file)):
        os.makedirs(os.path.dirname(pid_file), mode=0o755)
    file(pid_file, 'w+').write('%s\n' % pid)