def _handle_webhook_exception(self, webhook):
     exc = sys.exc_info()[1]
     if isinstance(exc, (
             requests.exceptions.Timeout,
             requests.exceptions.ProxyError,
             requests.exceptions.ConnectionError)):
         if webhook['handle_attempts'] < webhook['attempts']:
             webhook['status'] = 0
         else:
             webhook['status'] = 2
         webhook['error_msg'] = str(sys.exc_info()[0].__name__)
         msg = "Unable to process webhook: {0}, reason: {1}"
         msg = msg.format(webhook['history_id'], helper.exc_info())
         LOG.warning(msg)
     elif isinstance(exc, (
             requests.exceptions.RequestException,
             requests.packages.urllib3.exceptions.HTTPError,
             requests.packages.urllib3.exceptions.HTTPWarning)):
         webhook['status'] = 2
         webhook['error_msg'] = str(sys.exc_info()[0].__name__)
         msg = "Unable to process webhook: {0}, reason: {1}"
         msg = msg.format(webhook['history_id'], helper.exc_info())
         LOG.warning(msg)
     else:
         webhook['status'] = 2
         webhook['error_msg'] = 'Internal error'
         msg = "Unable to process webhook: {0}, reason: {1}"
         msg = msg.format(webhook['history_id'], helper.exc_info())
         LOG.error(msg)

    def _send(self, task):
        if not task:
            return
        try:
            msg = task['msg']
            req = task['req']
            try:

                LOG.debug('Send message %s host %s header %s'
                        % (msg['messageid'], req.get_host(), req.header_items()))
                code = urllib2.urlopen(
                    req, timeout=CONFIG['instances_connection_timeout']).getcode()
                if code != 201:
                    raise Exception('Server response code %s' % code)
                LOG.debug('Delivery ok, message %s, host %s'
                        % (msg['messageid'], req.get_host()))
                try:
                    self._db_update(True, msg)
                except:
                    LOG.error('Unable to update database %s' %helper.exc_info())
            except:
                e = sys.exc_info()[1]
                if type(e) in (urllib2.URLError, socket.timeout) and\
                        ('Connection refused' in str(e) or 'timed out' in str(e)):
                    LOG.warning('Delivery failed message id %s host %s error %s'
                            % (msg['messageid'], req.get_host(), helper.exc_info()))
                else:
                    LOG.error('Delivery failed message id %s host %s error %s'
                            % (msg['messageid'], req.get_host(), helper.exc_info()))
                self._db_update(False, msg)
        except:
            LOG.error(helper.exc_info())

 def run(self):
     while True:
         try:
             count = self._get_events_count()
             LOG.info(count)
             if count < CONFIG['warning_threshold']:
                 time.sleep(CONFIG['interval'])
                 continue
             message = 'DBQueueEvent alert. Events do not processed: %s' % count
             mail = MIMEText(message.encode('utf-8'), _charset='utf-8')
             mail['From'] = CONFIG['email_from']
             mail['To'] = CONFIG['email_to']
             if count > CONFIG['critical_threshold']:
                 mail['Subject'] = 'DBQueueEvent critical alert'
             else:
                 mail['Subject'] = 'DBQueueEvent warning alert'
             LOG.debug('Send mail\n%s' % mail.as_string())
             try:
                 server = smtplib.SMTP('localhost')
                 server.sendmail(mail['From'], mail['To'], mail.as_string())
             except:
                 LOG.error('Send mail fail: %s' % helper.exc_info())
             time.sleep(CONFIG['interval'])
         except KeyboardInterrupt:
             raise KeyboardInterrupt
         except:
             LOG.error(helper.exc_info())
             time.sleep(10)

    def _average(self, results, ra=None, fa=None, rs=None, fs=None):
        ra = ra or dict()
        fa = fa or dict()
        rs = rs or dict()
        fs = fs or dict()
        for result in results:
            try:
                r_key, f_key = self._get_rf_keys(result)
                if 'snum' in self.config['metrics']:
                    rs.setdefault(r_key, {'snum': {'s_running': 0}})
                    fs.setdefault(f_key, {'snum': {'s_running': 0}})
                    rs[r_key]['snum']['s_running'] += 1
                    fs[f_key]['snum']['s_running'] += 1
                if not result['data']:
                    continue
                for metrics_group_name, metrics_data in result['data'].iteritems():
                    if not metrics_data or metrics_group_name == 'io':
                        continue
                    for metric_name, value in metrics_data.iteritems():
                        try:
                            ra.setdefault(r_key, {})
                            ra[r_key].setdefault(metrics_group_name, {})
                            ra[r_key][metrics_group_name].setdefault(metric_name, Average())

                            fa.setdefault(f_key, {})
                            fa[f_key].setdefault(metrics_group_name, {})
                            fa[f_key][metrics_group_name].setdefault(metric_name, Average())

                            ra[r_key][metrics_group_name][metric_name] += value
                            fa[f_key][metrics_group_name][metric_name] += value
                        except:
                            LOG.error(helper.exc_info())
            except:
                LOG.error(helper.exc_info())
        return ra, fa, rs, fs

def get_metrics(host, port, key, api_type, metrics, headers=None, timeout=5):
    assert host, 'host'
    assert port, 'port'
    assert key, 'key'
    assert api_type, 'api_type'
    assert metrics, 'metrics'

    data = dict()
    endpoint = 'http://%s:%s' % (host, port)
    security = rpc.Security(key)
    hsp = rpc.HttpServiceProxy(endpoint, security=security, headers=headers)
    getters = {
        'cpu': get_cpu_stat,
        'la': get_la_stat,
        'mem': get_mem_info,
        'net': get_net_stat,
        'io': get_io_stat,
    }
    for metric in metrics:
        try:
            data.update({metric: getters[metric](hsp, api_type, timeout=timeout)})
        except (urllib2.URLError, urllib2.HTTPError, socket.timeout):
            msg = "Endpoint: {endpoint}, headers: {headers}, metric: '{metric}', reason: {err}"
            msg = msg.format(
                endpoint=endpoint, headers=headers, metric=metric, err=helper.exc_info())
            raise Exception(msg)
        except:
            msg = "Endpoint: {endpoint}, headers: {headers}, metric '{metric}' failed, reason: {er}"
            msg = msg.format(
                endpoint=endpoint, headers=headers, metric=metric, err=helper.exc_info())
            LOG.warning(msg)
            continue
    return data

    def do_iteration(self):
        self.iteration_timestamp = time.time()
        webhooks = self.get_webhooks()
        if not webhooks:
            raise NothingToDoError()

        for webhook in webhooks:
            try:
                wait_pool()
                webhook['async_result'] = POOL.apply_async(self.post_webhook, (webhook,))
            except:
                msg = "Unable to process webhook history_id: {0}, reason: {1}"
                msg = msg.format(webhook['history_id'], helper.exc_info())
                LOG.warning(msg)

        for webhook in webhooks:
            try:
                webhook['response_code'] = webhook['async_result'].get(timeout=60)
            except DBQueueEvent.PostError:
                error_msg = str(sys.exc_info()[1])
                self._handle_error(webhook, error_msg)
            except:
                error_msg = 'Internal error'
                self._handle_error(webhook, error_msg)
            try:
                wait_pool()
                POOL.apply_async(self.update_webhook, (webhook,))
            except:
                msg = "Unable to update webhook history_id: {0}, reason: {1}"
                msg = msg.format(webhook['history_id'], helper.exc_info())
                LOG.warning(msg)

        POOL.join()

 def rest_event_observer(self, event, config):
     try:
         key = "On%sNotifyURL" % event["type"]
         if key not in config or not config[key]:
             return
         payload = {"event": event["type"], "message": event["message"]}
         r = requests.post(config[key], params=payload, timeout=10)
         LOG.debug("Event:%s. Send request:'url:%s' status:'%s'" % (event["id"], config[key], r.status_code))
     except requests.exceptions.RequestException:
         LOG.warning(helper.exc_info())
     except:
         LOG.error(helper.exc_info())

 def rest_event_observer(self, event, config):
     try:
         key = 'On%sNotifyURL' % event['type']
         if key not in config or not config[key]:
             return
         payload = {'event': event['type'], 'message': event['message']}
         r = requests.post(config[key], params=payload, timeout=10)
         LOG.debug('Event:%s. Send request url:%s status %s' \
                 % (event['id'], config[key], r.status))
     except requests.exceptions.RequestException:
         LOG.warning(helper.exc_info())
     except:
         LOG.error(helper.exc_info())

    def __call__(self):
        poller_ps, plotter_ps = None, None

        if self.args['--plotter']:
            plotter = Plotter(self.config)
            plotter_ps = plotter.run_in_process()
            time.sleep(5)
            if not plotter_ps.is_alive():
                LOG.critical('Failed to start CherryPy web server')
                sys.exit(1)

        self.change_permissions()

        if self.args['--poller']:

            poller = Poller(self.config, self.scalr_config)
            while True:
                start_time = time.time()
                try:
                    LOG.info('Start poller iteration')

                    rrdcached_sock_file = self.config['rrd']['rrdcached_sock_path']
                    if not os.path.exists(rrdcached_sock_file):
                        raise Exception('rrdcached process is not running')

                    poller_ps = poller.run_in_process()
                    poller_ps.join(self.config['interval'] * 2)
                    if poller_ps.is_alive():
                        LOG.error('Poller iteration timeout. Terminating')
                        try:
                            poller_ps.terminate()
                        except:
                            msg = 'Unable to terminate, reason: {error}'.format(
                                error=helper.exc_info())
                            raise Exception(msg)
                    LOG.info('Poller iteration time: %.2f' % (time.time() - start_time))
                except KeyboardInterrupt:
                    raise
                except:
                    msg = 'Poller iteration failed, reason: {error}'.format(
                        error=helper.exc_info())
                    LOG.error(msg)
                finally:
                    sleep_time = start_time + self.config['interval'] - time.time() - 0.1
                    if sleep_time > 0:
                        time.sleep(sleep_time)

        if plotter_ps:
            plotter_ps.join()

 def update_server(self, server):
     try:
         szr_upd_client = self._get_szr_upd_client(server)
         timeout = self.config["instances_connection_timeout"]
         msg = "Trying to update server: {0}, version: {1}".format(server["server_id"], server["scalarizr.version"])
         LOG.debug(msg)
         try:
             result_id = szr_upd_client.update(async=True, timeout=timeout)
         except:
             msg = "Unable to update, reason: {0}".format(helper.exc_info())
             raise Exception(msg)
         LOG.debug("Server: {0}, result: {1}".format(server["server_id"], result_id))
     except:
         msg = "Server failed: {0}, reason: {1}".format(server["server_id"], helper.exc_info())
         LOG.warning(msg)

 def do_iteration(self):
     servers = self.get_servers_for_update()
     for server in servers:
         try:
             self._pool.wait()
             self._pool.apply_async(self.update_server, (server,))
             gevent.sleep(0)  # force switch
         except:
             LOG.warning(helper.exc_info())
     self._pool.join()
     try:
         self.update_scalr_repo_data()
     except:
         msg = "Unable to update scalr.settings table, reason: {0}".format(helper.exc_info())
         LOG.error(msg)

    def process_message(self, message, server):
        try:
            status = None
            try:
                request = self.make_request(message, server)
                if not request['url']:
                    msg = "Wrong request: {request}".format(request=request)
                    raise Exception(msg)
            except:
                self.update_error(message)
                raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]

            try:
                msg = "Send message: {message_id}, request: {request}"
                msg = msg.format(
                        message_id=message['messageid'],
                        request={'url': request['url'], 'headers': request['headers']})
                LOG.debug(msg)

                r = requests.post(
                        request['url'],
                        data=request['data'],
                        headers=request['headers'],
                        timeout=CONFIG['instances_connection_timeout'])

                if r.status_code != 201:
                    msg = "Bad response code: {code}".format(code=r.status_code)
                    raise Exception(msg)

                msg = "Message: {message_id}, delivery ok"
                msg = msg.format(message_id=message['messageid'])
                LOG.debug(msg)
                status = True
            except:
                msg = "Message: {message_id}, delivery failed, reason: {error}"
                msg = msg.format(message_id=message['messageid'], error=helper.exc_info())
                LOG.warning(msg)
                status = False

            if status:
                self.update_ok(message)
            else:
                self.update_not_ok(message)
        except:
            msg = "Unable to process message: {message_id}, server: {server}, reason: {error}"
            msg = msg.format(message_id=message['messageid'], server=server, error=helper.exc_info())
            LOG.warning(msg)
            raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]

    def _get_statuses(self, servers):
        async_results = {}
        for server in servers:
            if 'scalarizr.key' not in server:
                msg = "Server: {0}, reason: Missing scalarizr key".format(server['server_id'])
                LOG.warning(msg)
                continue
            if 'scalarizr.updc_port' not in server:
                api_port = self.scalr_config['scalarizr_update'].get('api_port', 8008)
                server['scalarizr.updc_port'] = api_port
            self._pool.wait()
            async_results[server['server_id']] = self._pool.apply_async(self._get_status, (server,))
            gevent.sleep(0)  # force switch

        statuses = {}
        timeout = self.config['instances_connection_timeout']
        for server in servers:
            try:
                server_id = server['server_id']
                statuses[server_id] = async_results[server_id].get(timeout=timeout)
            except:
                msg = 'Unable to get update client status, server: {0}, reason: {1}'
                msg = msg.format(server['server_id'], helper.exc_info())
                LOG.warning(msg)
        return statuses

 def encrypt_data(self, data):
     if not self.encrypt:
         return data
     try:
         return cryptotool.encrypt(self.crypto_algo, data, self.crypto_key)
     except:
         raise InvalidRequestError('Failed to encrypt data. Error:%s' % helper.exc_info())

def _ec2_region(region, cred):
    try:
        access_key = cryptotool.decrypt_scalr(app.crypto_key, cred["access_key"])
        secret_key = cryptotool.decrypt_scalr(app.crypto_key, cred["secret_key"])
        kwds = {"aws_access_key_id": access_key, "aws_secret_access_key": secret_key}
        proxy_settings = app.proxy_settings.get(cred.platform, {})
        kwds["proxy"] = proxy_settings.get("host")
        kwds["proxy_port"] = proxy_settings.get("port")
        kwds["proxy_user"] = proxy_settings.get("user")
        kwds["proxy_pass"] = proxy_settings.get("pass")

        msg = "List nodes for platform: 'ec2', region: '{}', envs_ids: {}"
        msg = msg.format(region, cred.envs_ids)
        LOG.debug(msg)

        conn = boto.ec2.connect_to_region(region, **kwds)
        cloud_nodes = _ec2_get_only_instances(conn)
        timestamp = int(time.time())
        nodes = list()
        for cloud_node in cloud_nodes:
            node = {
                "instance_id": cloud_node.id,
                "instance_type": cloud_node.instance_type,
                "os": cloud_node.platform if cloud_node.platform else "linux",
            }
            nodes.append(node)
        return {"region": region, "timestamp": timestamp, "nodes": nodes} if nodes else dict()
    except:
        e = sys.exc_info()[1]
        msg = "platform: '{platform}', region: '{region}', envs_ids: {envs_ids}. Reason: {error}"
        msg = msg.format(
            platform=cred.platform, region=region, envs_ids=cred.envs_ids, error=helper.exc_info(where=False)
        )
        _handle_exception(e, msg)

def farm_process(tasks):
    if not tasks:
        return

    try:
        servs_pool = pool.ThreadPool(processes=CONFIG['serv_thrds'])
        rrd_pool = pool.ThreadPool(processes=CONFIG['rrd_thrds'])
        results = servs_pool.map(server_thread, [(t, rrd_pool) for t in tasks])
        servs_pool.close()

        if not results:
            return

        ra, fa, rs, fs = post_processing(results)

        for k, v in ra.iteritems():
            rrd_pool.map_async(RRDWorker().work, [{'ra':{k:v}}])

        for k, v in fa.iteritems():
            rrd_pool.map_async(RRDWorker().work, [{'fa':{k:v}}])

        for k, v in rs.iteritems():
            rrd_pool.map_async(RRDWorker().work, [{'rs':{k:v}}])

        for k, v in fs.iteritems():
            rrd_pool.map_async(RRDWorker().work, [{'fs':{k:v}}])
    except:
        LOG.error(helper.exc_info())
    finally:
        servs_pool.close()
        servs_pool.join()
        rrd_pool.close()
        rrd_pool.join()

def main():
    parser = argparse.ArgumentParser()

    group1 = parser.add_mutually_exclusive_group()
    group1.add_argument("--start", action="store_true", default=False, help="start program")
    group1.add_argument("--stop", action="store_true", default=False, help="stop program")
    parser.add_argument("--no-daemon", action="store_true", default=None, help="run in no daemon mode")
    parser.add_argument("-p", "--pid-file", default=None, help="pid file")
    parser.add_argument("-l", "--log-file", default=None, help="log file")
    parser.add_argument("-c", "--config-file", default="./config.yml", help="config file")
    parser.add_argument("-v", "--verbosity", action="count", default=None, help="increase output verbosity")
    parser.add_argument("--version", action="version", version="Version %s" % __version__)
    parser.add_argument("--recalculate", action="store_true", default=False, help="recalculate data")
    parser.add_argument(
        "--platform",
        type=str,
        default=False,
        help=(
            "platform to recalculate, "
            "[cloudstack, ec2, ecs, eucalyptus, gce, idcf, openstack, "
            "rackspacenguk, rackspacengus]"
        ),
    )
    parser.add_argument("--date-from", type=str, default=False, help="from date, 'YYYY-MM-DD' UTC")
    parser.add_argument("--date-to", type=str, default=False, help="to date, 'YYYY-MM-DD' UTC")

    args = parser.parse_args()
    try:
        config = yaml.safe_load(open(args.config_file))["scalr"]
        configure(config, args)
    except SystemExit:
        raise
    except:
        if args.verbosity > 3:
            raise
        else:
            sys.stderr.write("%s\n" % helper.exc_info(line_no=False))
        sys.exit(1)
    try:
        app = AnalyticsProcessing()
        if args.start:
            if helper.check_pid(CONFIG["pid_file"]):
                msg = "Application with pid file '%s' already running. Exit" % CONFIG["pid_file"]
                LOG.info(msg)
                sys.exit(0)
            if not args.no_daemon:
                helper.daemonize()
            app.start()
        elif args.stop:
            app.stop()
        else:
            print "Usage %s -h" % sys.argv[0]
    except KeyboardInterrupt:
        LOG.critical("KeyboardInterrupt")
        return
    except SystemExit:
        pass
    except:
        LOG.exception("Something happened and I think I died")
        sys.exit(1)

 def _get_servers(self, farms):
     for servers in self._get_db_servers(farms):
         servers_properties = self._get_db_servers_properties(servers)
         servers_vpc_ip = self._db.get_servers_vpc_ip(servers)
         farms_hash = self._get_farms_hash(farms)
         envs_status = self._db.get_envs_status_by_servers(servers)
         out = []
         for server in servers:
             try:
                 if envs_status[server['env_id']] != 'Active':
                     continue
                 server_id = server['server_id']
                 server['server_properties'] = servers_properties[server_id]
                 server['farm_hash'] = farms_hash[server['farm_id']]
                 if server_id in servers_vpc_ip:
                     server['vpc_ip'] = servers_vpc_ip[server_id]
                 if server['os_type'] == 'linux':
                     exclude = ['snum']
                 elif server['os_type'] == 'windows':
                     exclude = ['la', 'io', 'snum']
                 else:
                     msg = 'Wrong os type for server %s' % server['server_id']
                     raise Exception(msg)
                 metrics = [m for m in CONFIG['metrics'] if m not in exclude]
                 server['metrics'] = metrics
                 out.append(server)
             except:
                 LOG.error(helper.exc_info())
                 continue
         yield out

    def get_prices(self, servers):
        """
        :returns: dict {account_id: {platform_url: {cloud_location: {instance_type: {os: cost}}}}}
        """

        prices = dict()
        for raw_prices in self._get_raw_prices(servers):
            for raw_price in raw_prices:
                try:
                    account_id = raw_price['account_id']
                    platform = raw_price['platform']
                    url = raw_price['url']
                    platform_url = '%s;%s' % (platform, url)
                    cloud_location = raw_price['cloud_location']
                    instance_type = raw_price['instance_type']
                    os = raw_price['os']
                    cost = raw_price['cost']
                    prices.setdefault(account_id, dict())
                    prices[account_id].setdefault(platform_url, dict())
                    prices[account_id][platform_url].setdefault(cloud_location, dict())
                    prices[account_id][platform_url][cloud_location].setdefault(instance_type, dict())
                    prices[account_id][platform_url][cloud_location][instance_type][os] = cost
                except KeyError:
                    msg = "Unable to get price from raw price, reason: {error}"
                    msg = msg.format(error=helper.exc_info())
                    LOG.warning(msg)
        return prices

 def __init__(self, record=None):
     Table.__init__(self)
     self._types = {
             'usage_id': UUIDType,
             'dtime': QuoteType,
             'platform': QuoteType,
             'url': QuoteType,
             'cloud_location': QuoteType,
             'instance_type': QuoteType,
             'os': NoQuoteType,
             'num': NoQuoteType,
             'cost': NoQuoteType,
     }
     self._fill(record)
     if 'usage_id' not in self:
         try:
             formatted = self._format()
             unique = '; '.join(
                 [
                     str(formatted['dtime']).strip(), str(formatted['platform']).strip(),
                     str(formatted['url']).strip(), str(formatted['cloud_location']).strip(),
                     str(formatted['instance_type']).strip(), str(formatted['os']).strip(),
                 ]
             )
             self['usage_id'] = uuid.uuid5(UUID, unique).hex
         except KeyError:
             msg = "Can't set not managed usage_id for record: {record}, reason: {error}"
             msg = msg.format(record=record, error=helper.exc_info())
             LOG.warning(msg)