def _process_ping(self, ping_url, ping_reply, auth, tags, pool_name, http_host):
        if ping_reply is None:
            ping_reply = 'pong'

        sc_tags = ["ping_url:{0}".format(ping_url)] + tags
        if http_host is not None:
            sc_tags += ["http_host:{0}".format(http_host)]

        try:
            # TODO: adding the 'full' parameter gets you per-process detailed
            # informations, which could be nice to parse and output as metrics
            resp = requests.get(ping_url, auth=auth,
                                headers=headers(self.agentConfig, http_host=http_host))
            resp.raise_for_status()

            if ping_reply not in resp.text:
                raise Exception("Received unexpected reply to ping {0}".format(resp.text))

        except Exception as e:
            self.log.error("Failed to ping FPM pool {0} on URL {1}."
                           "\nError {2}".format(pool_name, ping_url, e))
            self.service_check(self.SERVICE_CHECK_NAME,
                               AgentCheck.CRITICAL, tags=sc_tags, message=str(e))
        else:
            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=sc_tags)

    def _process_status(self, status_url, auth, tags):
        data = {}
        try:
            # TODO: adding the 'full' parameter gets you per-process detailed
            # informations, which could be nice to parse and output as metrics
            resp = requests.get(status_url, auth=auth,
                                headers=headers(self.agentConfig),
                                params={'json': True})
            resp.raise_for_status()

            data = resp.json()
        except Exception as e:
            self.log.error("Failed to get metrics from {0}.\nError {1}".format(status_url, e))
            raise

        pool_name = data.get('pool', 'default')
        metric_tags = tags + ["pool:{0}".format(pool_name)]

        for key, mname in self.GAUGES.iteritems():
            if key not in data:
                self.log.warn("Gauge metric {0} is missing from FPM status".format(key))
                continue
            self.gauge(mname, int(data[key]), tags=metric_tags)

        for key, mname in self.MONOTONIC_COUNTS.iteritems():
            if key not in data:
                self.log.warn("Counter metric {0} is missing from FPM status".format(key))
                continue
            self.monotonic_count(mname, int(data[key]), tags=metric_tags)

        # return pool, to tag the service check with it if we have one
        return pool_name

    def _get_data(self, url, config, send_sc=True):
        """ Hit a given URL and return the parsed json
        """
        # Load basic authentication configuration, if available.
        if config.username and config.password:
            auth = (config.username, config.password)
        else:
            auth = None

        try:
            resp = requests.get(
                url,
                timeout=config.timeout,
                headers=headers(self.agentConfig),
                auth=auth
            )
            resp.raise_for_status()
        except Exception as e:
            if send_sc:
                self.service_check(
                    self.SERVICE_CHECK_CONNECT_NAME,
                    AgentCheck.CRITICAL,
                    message="Error {0} when hitting {1}".format(e, url),
                    tags=config.service_check_tags
                )
            raise

        return resp.json()

    def check(self, instance):
        if "monitor_agent_url" not in instance:
            raise Exception('Fluentd instance missing "monitor_agent_url" value.')

        try:
            url = instance.get("monitor_agent_url")
            plugin_ids = instance.get("plugin_ids", [])

            parsed_url = urlparse.urlparse(url)
            monitor_agent_host = parsed_url.hostname
            monitor_agent_port = parsed_url.port or 24220
            service_check_tags = ["fluentd_host:%s" % monitor_agent_host, "fluentd_port:%s" % monitor_agent_port]

            req = urllib2.Request(url, None, headers(self.agentConfig))
            res = urllib2.urlopen(req).read()
            status = json.loads(res)

            for p in status["plugins"]:
                for m in self.GAUGES:
                    if p.get(m) is None:
                        continue
                    if p.get("plugin_id") in plugin_ids:
                        self.gauge("fluentd.%s" % (m), p.get(m), ["plugin_id:%s" % p.get("plugin_id")])
        except Exception, e:
            msg = "No stats could be retrieved from %s : %s" % (url, str(e))
            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=msg)
            raise e

    def _fetch_data(self, instance):
        if 'kong_status_url' not in instance:
            raise Exception('missing "kong_status_url" value')
        tags = instance.get('tags', [])
        url = instance.get('kong_status_url')

        parsed_url = urlparse.urlparse(url)
        host = parsed_url.hostname
        port = parsed_url.port
        service_check_name = 'kong.can_connect'
        service_check_tags = ['kong_host:%s' % host, 'kong_port:%s' % port]

        try:
            self.log.debug(u"Querying URL: {0}".format(url))
            response = requests.get(url, headers=headers(self.agentConfig))
            self.log.debug(u"Kong status `response`: {0}".format(response))
            response.raise_for_status()
        except Exception:
            self.service_check(service_check_name, AgentCheck.CRITICAL,
                               tags=service_check_tags)
            raise
        else:
            if response.status_code == 200:
                self.service_check(service_check_name, AgentCheck.OK,
                               tags=service_check_tags)
            else:
                self.service_check(service_check_name, AgentCheck.CRITICAL,
                               tags=service_check_tags)

        return self._parse_json(response.content, tags)

    def _get_data(self, instance):
        url = instance.get("nginx_status_url")
        ssl_validation = instance.get("ssl_validation", True)

        auth = None
        if "user" in instance and "password" in instance:
            auth = (instance["user"], instance["password"])

        # Submit a service check for status page availability.
        parsed_url = urlparse.urlparse(url)
        nginx_host = parsed_url.hostname
        nginx_port = parsed_url.port or 80
        service_check_name = "nginx.can_connect"
        service_check_tags = ["host:%s" % nginx_host, "port:%s" % nginx_port]
        try:
            self.log.debug(u"Querying URL: {0}".format(url))
            r = requests.get(url, auth=auth, headers=headers(self.agentConfig), verify=ssl_validation)
            r.raise_for_status()
        except Exception:
            self.service_check(service_check_name, AgentCheck.CRITICAL, tags=service_check_tags)
            raise
        else:
            self.service_check(service_check_name, AgentCheck.OK, tags=service_check_tags)

        body = r.content
        resp_headers = r.headers
        return body, resp_headers.get("content-type", "text/plain")

    def _get_data(self, instance):
        url = instance.get('nginx_status_url')

        auth = None
        if 'user' in instance and 'password' in instance:
            auth = (instance['user'], instance['password'])

        # Submit a service check for status page availability.
        parsed_url = urlparse.urlparse(url)
        nginx_host = parsed_url.hostname
        nginx_port = parsed_url.port or 80
        service_check_name = 'nginx.can_connect'
        service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]
        try:
            r = requests.get(url, auth=auth, headers=headers(self.agentConfig))
            r.raise_for_status()
        except Exception:
            self.service_check(service_check_name, AgentCheck.CRITICAL,
                               tags=service_check_tags)
            raise
        else:
            self.service_check(service_check_name, AgentCheck.OK,
                               tags=service_check_tags)

        body = r.content
        resp_headers = r.headers
        return body, resp_headers.get('content-type', 'text/plain')

    def check(self, logger, agentConfig):

        if 'rabbitMQStatusUrl' not in agentConfig or \
           'rabbitMQUser' not in agentConfig or \
           'rabbitMQPass' not in agentConfig or \
            agentConfig['rabbitMQStatusUrl'] == 'http://www.example.com:55672/json':
            return False

        try:
            logger.debug('getRabbitMQStatus: attempting authentication setup')
            manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            manager.add_password(None, agentConfig['rabbitMQStatusUrl'], agentConfig['rabbitMQUser'], agentConfig['rabbitMQPass'])
            handler = urllib2.HTTPBasicAuthHandler(manager)
            opener = urllib2.build_opener(handler)
            urllib2.install_opener(opener)

            logger.debug('getRabbitMQStatus: attempting urlopen')
            req = urllib2.Request(agentConfig['rabbitMQStatusUrl'], None, headers(agentConfig))

            # Do the request, log any errors
            request = urllib2.urlopen(req)
            response = request.read()

            return json.loads(response)
        except:
            logger.exception('Unable to get RabbitMQ status')
            return False

    def check(self, instance):
        if 'apache_status_url' not in instance:
            self.log.warn("Missing 'apache_status_url' in Apache config")
            return
        tags = instance.get('tags', [])

        try:
            req = urllib2.Request(instance['apache_status_url'], None,
                headers(self.agentConfig))
            request = urllib2.urlopen(req)
            response = request.read()

            # Loop through and extract the numerical values
            for line in response.split('\n'):
                values = line.split(': ')
                if len(values) == 2: # match
                    metric, value = values
                    metric_name = self.METRIC_TRANSLATION.get(metric, metric)
                    try:
                        if metric_name == 'apache.net.bytes':
                            self.gauge(metric_name, float(value) * 1024, tags=tags)
                        else:
                            self.gauge(metric_name, float(value), tags=tags)
                    except ValueError:
                        continue
        except:
            self.log.exception('Unable to get Apache status')

    def check(self, instance):
        if 'lighttpd_status_url' not in instance:
            raise Exception("Missing 'lighttpd_status_url' in Lighttpd config")

        tags = instance.get('tags', [])
        req = urllib2.Request(instance['lighttpd_status_url'], None,
            headers(self.agentConfig))
        request = urllib2.urlopen(req)
        response = request.read()

        # Loop through and extract the numerical values
        for line in response.split('\n'):
            values = line.split(': ')
            if len(values) == 2: # match
                metric, value = values
                try:
                    value = float(value)
                except ValueError:
                    continue

                # Special case: kBytes => bytes
                if metric == 'Total kBytes':
                    value = value * 1024

                # Send metric as a gauge, if applicable
                if metric in self.GAUGES:
                    metric_name = self.GAUGES[metric]
                    self.gauge(metric_name, value, tags=tags)

                # Send metric as a rate, if applicable
                if metric in self.RATES:
                    metric_name = self.RATES[metric]
                    self.rate(metric_name, value, tags=tags)

    def check(self, instance):
        if 'monitor_agent_url' not in instance:
            raise Exception('Fluentd instance missing "monitor_agent_url" value.')

        try:
            url = instance.get('monitor_agent_url')
            plugin_ids = instance.get('plugin_ids', [])

            parsed_url = urlparse.urlparse(url)
            monitor_agent_host = parsed_url.hostname
            monitor_agent_port = parsed_url.port or 24220
            service_check_tags = ['fluentd_host:%s' % monitor_agent_host, 'fluentd_port:%s' % monitor_agent_port]

            r = requests.get(url, headers=headers(self.agentConfig))
            r.raise_for_status()
            status = r.json()

            for p in status['plugins']:
                for m in self.GAUGES:
                    if p.get(m) is None:
                        continue
                    if p.get('plugin_id') in plugin_ids:
                        self.gauge('fluentd.%s' % (m), p.get(m), ["plugin_id:%s" % p.get('plugin_id')])
        except Exception, e:
            msg = "No stats could be retrieved from %s : %s" % (url, str(e))
            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=msg)
            raise e

    def check(self, instance):
        if 'gostats_url' not in instance:
            raise Exception("Missing 'gostats_url' in GoStats config")

        try:
            url = instance.get('gostats_url')
            parsed_url = urlparse.urlparse(url)
            gostats_host = parsed_url.hostname
            gostats_port = parsed_url.port or 8080

            r = requests.get(url, headers=headers(self.agentConfig))
            r.raise_for_status()
            status = r.json()

            for metric in self.GAUGES:
                if status.get(metric) is None:
                    continue
                self.gauge('gostats.%s' % (metric), status.get(metric))

            for metric in self.HISTGRAMS:
                if status.get(metric) is None:
                    continue
                for value in status.get(metric):
                    self.histogram('gostats.%s' % (metric), value)

        except Exception, e:
            self.log.error('error: %s' % str(e))
            raise

    def _get_data(self, instance):
        url = instance.get('nginx_status_url')
        ssl_validation = instance.get('ssl_validation', True)

        auth = None
        if 'user' in instance and 'password' in instance:
            auth = (instance['user'], instance['password'])

        # Submit a service check for status page availability.
        parsed_url = urlparse.urlparse(url)
        nginx_host = parsed_url.hostname
        nginx_port = parsed_url.port or 80
        service_check_name = 'nginx.can_connect'
        service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]
        try:
            self.log.debug(u"Querying URL: {0}".format(url))
            r = requests.get(url, auth=auth, headers=headers(self.agentConfig),
                             verify=ssl_validation, timeout=self.default_integration_http_timeout)
            r.raise_for_status()
        except Exception:
            self.service_check(service_check_name, AgentCheck.CRITICAL,
                               tags=service_check_tags)
            raise
        else:
            self.service_check(service_check_name, AgentCheck.OK,
                               tags=service_check_tags)

        body = r.content
        resp_headers = r.headers
        return body, resp_headers.get('content-type', 'text/plain')

    def _get_data(self, instance):
        url = instance.get('nginx_status_url')
        req = urllib2.Request(url, None, headers(self.agentConfig))
        if 'user' in instance and 'password' in instance:
            add_basic_auth(req, instance['user'], instance['password'])

        # Submit a service check for status page availability.
        parsed_url = urlparse.urlparse(url)
        nginx_host = parsed_url.hostname
        nginx_port = parsed_url.port or 80
        service_check_name = 'nginx.can_connect'
        service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]
        try:
            response = urllib2.urlopen(req)
        except Exception:
            self.service_check(service_check_name, AgentCheck.CRITICAL,
                               tags=service_check_tags)
            raise
        else:
            self.service_check(service_check_name, AgentCheck.OK,
                               tags=service_check_tags)

        body = response.read()
        resp_headers = response.info()
        return body, resp_headers.get('Content-Type', 'text/plain')

    def check(self, instance):
        if 'monitor_agent_url' not in instance:
            raise Exception('Fluentd instance missing "monitor_agent_url" value.')

        try:
            url = instance.get('monitor_agent_url')
            plugin_ids = instance.get('plugin_ids', [])

            # Fallback  with `tag_by: plugin_id`
            tag_by = instance.get('tag_by')
            tag_by = tag_by if tag_by in self._AVAILABLE_TAGS else 'plugin_id'

            parsed_url = urlparse.urlparse(url)
            monitor_agent_host = parsed_url.hostname
            monitor_agent_port = parsed_url.port or 24220
            service_check_tags = ['fluentd_host:%s' % monitor_agent_host, 'fluentd_port:%s'
                                  % monitor_agent_port]

            r = requests.get(url, headers=headers(self.agentConfig))
            r.raise_for_status()
            status = r.json()

            for p in status['plugins']:
                tag = "%s:%s" % (tag_by, p.get(tag_by))
                for m in self.GAUGES:
                    if p.get(m) is None:
                        continue
                    # Filter unspecified plugins to keep backward compatibility.
                    if len(plugin_ids) == 0 or p.get('plugin_id') in plugin_ids:
                        self.gauge('fluentd.%s' % (m), p.get(m), [tag])
        except Exception, e:
            msg = "No stats could be retrieved from %s : %s" % (url, str(e))
            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL,
                               tags=service_check_tags, message=msg)
            raise

 def _get_data(self, instance):
     url = instance.get('nginx_status_url')
     req = urllib2.Request(url, None, headers(self.agentConfig))
     if 'user' in instance and 'password' in instance:
         add_basic_auth(req, instance['user'], instance['password'])
     request = urllib2.urlopen(req)
     return request.read()

    def _get_metrics(self, url, tags):
        req = urllib2.Request(url, None, headers(self.agentConfig))
        request = urllib2.urlopen(req)
        response = request.read()

        # Thanks to http://hostingfu.com/files/nginx/nginxstats.py for this code
        # Connections
        parsed = re.search(r'Active connections:\s+(\d+)', response)
        if parsed:
            connections = int(parsed.group(1))
            self.gauge("nginx.net.connections", connections, tags=tags)

        # Requests per second
        parsed = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', response)
        if parsed:
            conn = int(parsed.group(1))
            requests = int(parsed.group(3))
            self.rate("nginx.net.conn_opened_per_s", conn, tags=tags)
            self.rate("nginx.net.request_per_s", requests, tags=tags)

        # Connection states, reading, writing or waiting for clients
        parsed = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', response)
        if parsed:
            reading, writing, waiting = map(int, parsed.groups())
            self.gauge("nginx.net.reading", reading, tags=tags)
            self.gauge("nginx.net.writing", writing, tags=tags)
            self.gauge("nginx.net.waiting", waiting, tags=tags)

    def check(self, instance):
        if 'lighttpd_status_url' not in instance:
            raise Exception("Missing 'lighttpd_status_url' variable in Lighttpd config")

        url = self.assumed_url.get(instance['lighttpd_status_url'], instance['lighttpd_status_url'])

        tags = instance.get('tags', [])
        self.log.debug("Connecting to %s" % url)
        req = urllib2.Request(url, None, headers(self.agentConfig))
        if 'user' in instance and 'password' in instance:
            add_basic_auth(req, instance['user'], instance['password'])
        request = urllib2.urlopen(req)
        headers_resp = request.info().headers
        server_version = self._get_server_version(headers_resp)
        response = request.read()

        metric_count = 0
        # Loop through and extract the numerical values
        for line in response.split('\n'):
            values = line.split(': ')
            if len(values) == 2: # match
                metric, value = values
                try:
                    value = float(value)
                except ValueError:
                    continue

                # Special case: kBytes => bytes
                if metric == 'Total kBytes':
                    value = value * 1024

                # Send metric as a gauge, if applicable
                if metric in self.GAUGES:
                    metric_count += 1
                    metric_name = self.GAUGES[metric]
                    self.gauge(metric_name, value, tags=tags)

                # Send metric as a rate, if applicable
                if metric in self.RATES:
                    metric_count += 1
                    metric_name = self.RATES[metric]
                    self.rate(metric_name, value, tags=tags)

                # Send metric as a counter, if applicable
                if metric in self.COUNTERS:
                    metric_count += 1
                    metric_name = self.COUNTERS[metric]
                    self.increment(metric_name, value, tags=tags)

        if metric_count == 0:
            url_suffix = self.URL_SUFFIX_PER_VERSION[server_version]
            if self.assumed_url.get(instance['lighttpd_status_url'], None) is None and url[-len(url_suffix):] != url_suffix:
                self.assumed_url[instance['lighttpd_status_url']] = '%s%s' % (url, url_suffix)
                self.warning("Assuming url was not correct. Trying to add %s suffix to the url" % url_suffix)
                self.check(instance)
            else:
                raise Exception("No metrics were fetched for this instance. Make sure that %s is the proper url." % instance['lighttpd_status_url'])

 def _get_data(self, instance):
     url = instance.get('nginx_status_url')
     req = urllib2.Request(url, None, headers(self.agentConfig))
     if 'user' in instance and 'password' in instance:
         add_basic_auth(req, instance['user'], instance['password'])
     response = urllib2.urlopen(req)
     body = response.read()
     resp_headers = response.info()
     return body, resp_headers.get('Content-Type', 'text/plain')

    def _get_stats(self, url):
        "Hit a given URL and return the parsed json"
        self.log.debug('Fetching Couchbase stats at url: %s' % url)
        req = urllib2.Request(url, None, headers(self.agentConfig))

        # Do the request, log any errors
        request = urllib2.urlopen(req)
        response = request.read()
        return json.loads(response)