func (n *Namespace) NewLabeledTimer(name, help string, labels ...string) LabeledTimer {
	t := &labeledTimer{
		m: prometheus.NewHistogramVec(n.newTimerOpts(name, help), labels),
	}
	n.addMetric(t)
	return t
}

// NewMiddleware returns a new prometheus Middleware handler.
func NewMiddleware(name string, buckets ...float64) *Middleware {
	var m Middleware
	m.reqs = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name:        reqsName,
			Help:        "How many HTTP requests processed, partitioned by status code, method and HTTP path.",
			ConstLabels: prometheus.Labels{"service": name},
		},
		[]string{"code", "method", "path"},
	)
	prometheus.MustRegister(m.reqs)

	if len(buckets) == 0 {
		buckets = dflBuckets
	}
	m.latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:        latencyName,
		Help:        "How long it took to process the request, partitioned by status code, method and HTTP path.",
		ConstLabels: prometheus.Labels{"service": name},
		Buckets:     buckets,
	},
		[]string{"code", "method", "path"},
	)
	prometheus.MustRegister(m.latency)
	return &m
}

// NewHistogram returns a new Histogram backed by a Prometheus Histogram. The
// histogram is automatically registered via prometheus.Register.
//
// For more information on Prometheus histograms and summaries, refer to
// http://prometheus.io/docs/practices/histograms.
func NewHistogram(opts prometheus.HistogramOpts, fieldKeys []string) metrics.Histogram {
	m := prometheus.NewHistogramVec(opts, fieldKeys)
	prometheus.MustRegister(m)
	return prometheusHistogram{
		HistogramVec: m,
		Pairs:        pairsFrom(fieldKeys),
	}
}

func defineMetrics(namespace, subsystem string) {
	requestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      "dns_request_count_total",
		Help:      "Counter of DNS requests made.",
	}, []string{"system"})

	requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      "dns_request_duration_seconds",
		Help:      "Histogram of the time (in seconds) each request took to resolve.",
		Buckets:   append([]float64{0.001, 0.003}, prometheus.DefBuckets...),
	}, []string{"system"})

	responseSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      "dns_response_size_bytes",
		Help:      "Size of the returns response in bytes.",
		Buckets: []float64{0, 512, 1024, 1500, 2048, 4096,
			8192, 12288, 16384, 20480, 24576, 28672, 32768, 36864,
			40960, 45056, 49152, 53248, 57344, 61440, 65536,
		},
	}, []string{"system"})

	errorCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      "dns_error_count_total",
		Help:      "Counter of DNS requests resulting in an error.",
	}, []string{"system", "cause"})

	cacheMiss = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      "dns_cachemiss_count_total",
		Help:      "Counter of DNS requests that result in a cache miss.",
	}, []string{"cache"})
}

func TestWriteHistogram(t *testing.T) {
	histVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:        "name",
			Help:        "docstring",
			ConstLabels: prometheus.Labels{"constname": "constvalue"},
			Buckets:     []float64{0.01, 0.02, 0.05, 0.1},
		},
		[]string{"labelname"},
	)

	histVec.WithLabelValues("val1").Observe(float64(10))
	histVec.WithLabelValues("val1").Observe(float64(20))
	histVec.WithLabelValues("val1").Observe(float64(30))
	histVec.WithLabelValues("val2").Observe(float64(20))
	histVec.WithLabelValues("val2").Observe(float64(30))
	histVec.WithLabelValues("val2").Observe(float64(40))

	reg := prometheus.NewRegistry()
	reg.MustRegister(histVec)

	mfs, err := reg.Gather()
	if err != nil {
		t.Fatalf("error: %v", err)
	}

	now := model.Time(1477043083)
	var buf bytes.Buffer
	err = writeMetrics(&buf, mfs, "prefix", now)
	if err != nil {
		t.Fatalf("error: %v", err)
	}

	want := `prefix.name_bucket.constname.constvalue.labelname.val1.le.0_01 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val1.le.0_02 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val1.le.0_05 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val1.le.0_1 0 1477043
prefix.name_sum.constname.constvalue.labelname.val1 60 1477043
prefix.name_count.constname.constvalue.labelname.val1 3 1477043
prefix.name_bucket.constname.constvalue.labelname.val1.le._Inf 3 1477043
prefix.name_bucket.constname.constvalue.labelname.val2.le.0_01 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val2.le.0_02 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val2.le.0_05 0 1477043
prefix.name_bucket.constname.constvalue.labelname.val2.le.0_1 0 1477043
prefix.name_sum.constname.constvalue.labelname.val2 90 1477043
prefix.name_count.constname.constvalue.labelname.val2 3 1477043
prefix.name_bucket.constname.constvalue.labelname.val2.le._Inf 3 1477043
`
	if got := buf.String(); want != got {
		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
	}
}

// EnableHandlingTimeHistogram turns on recording of handling time of RPCs for server-side interceptors.
// Histogram metrics can be very expensive for Prometheus to retain and query.
func EnableHandlingTimeHistogram(opts ...HistogramOption) {
	for _, o := range opts {
		o(&serverHandledHistogramOpts)
	}
	if !serverHandledHistogramEnabled {
		serverHandledHistogram = prom.NewHistogramVec(
			serverHandledHistogramOpts,
			[]string{"grpc_type", "grpc_service", "grpc_method"},
		)
		prom.Register(serverHandledHistogram)
	}
	serverHandledHistogramEnabled = true
}

// Histogram implements xstats.Sender interface
//
// Mark the tags as "key:value".
func (s sender) Histogram(stat string, value float64, tags ...string) {
	s.RLock()
	m, ok := s.histograms[stat]
	s.RUnlock()
	keys, values := splitTags(tags)
	if !ok {
		s.Lock()
		if m, ok = s.histograms[stat]; !ok {
			m = prometheus.NewHistogramVec(
				prometheus.HistogramOpts{Name: stat, Help: stat},
				keys)
			prometheus.MustRegister(m)
			s.histograms[stat] = m
		}
		s.Unlock()
	}
	m.WithLabelValues(values...).Observe(value)
}

func newMeasurer() grpcinstrument.Measurer {
	return &measurer{
		registry: &registry{
			total: prometheus.NewCounterVec(prometheus.CounterOpts{
				Name: "grpc_calls_total",
				Help: "Number of gRPC calls received by the server being instrumented.",
			}, []string{"service", "method"}),
			errors: prometheus.NewCounterVec(prometheus.CounterOpts{
				Name: "grpc_calls_errors",
				Help: "Number of gRPC calls that returned an error.",
			}, []string{"service", "method"}),
			duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
				Name: "grpc_calls_durations",
				Help: "Duration of gRPC calls.",
			}, []string{"service", "method"}),
		},
	}
}

func main() {
	var (
		listen  = flag.String("listen", ":7800", "Server listen address.")
		name    = flag.String("test.name", "unknown", "Name of the test to run.")
		rate    = flag.Uint64("test.rate", defaultRate, "Number of requests to send during test duration.")
		timeout = flag.Duration("test.timeout", defaultTimeout, "Time until a request is discarded")

		ts = targets{}
	)
	flag.Var(&ts, "test.target", `Target to hit by the test with the following format: -test.target="NAME:address/url"`)
	flag.Parse()

	if *listen == "" || len(ts) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	latencies := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace:   "eagle",
			Name:        "request_duration_seconds",
			Help:        "The total duration of HTTP requests (seconds).",
			ConstLabels: prometheus.Labels{"test": *name},
			Buckets:     []float64{.001, .002, .003, .004, .005, .0075, .01, .015},
		},
		[]string{"target", "endpoint", "code"},
	)
	prometheus.MustRegister(latencies)

	resultc := make(chan result)
	go func() {
		for r := range resultc {
			latencies.
				WithLabelValues(r.target, r.endpoint, strconv.Itoa(int(r.Code))).
				Observe(r.Latency.Seconds())
		}
	}()
	newTest(*name, *rate, *timeout, ts).run(resultc)

	http.Handle("/metrics", prometheus.Handler())
	log.Printf("Starting server on %s", *listen)
	log.Fatal(http.ListenAndServe(*listen, nil))
}

func (p *prometheus) Histogram(id string) metrics.Histogram {
	var fields []string
	for k, _ := range p.opts.Fields {
		fields = append(fields, k)
	}

	hv := pr.NewHistogramVec(pr.HistogramOpts{
		Namespace: format(p.opts.Namespace),
		Name:      format(id),
		Help:      "histogram",
	}, fields)

	p.buf <- hv

	return &histogram{
		id: id,
		hv: hv,
		f:  p.opts.Fields,
	}
}

func (s *Sink) emitHistogram(job string, event string, nanos int64, kvs map[string]string) {
	var counter *prometheus.HistogramVec
	if c, ok := s.Config.HistogramVecs[job]; ok {
		counter = c
	} else {
		labels := labelsFromMap(kvs)

		counter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
			Namespace: s.Config.Job,
			Subsystem: "duration",
			Name:      job,
			Help:      "Automaticaly created event",
		}, labels)

		s.Config.HistogramVecs[job] = counter
		prometheus.Register(counter)
	}

	kvs["event"] = event

	if m, err := counter.GetMetricWith(kvs); err == nil {
		m.Observe(float64(nanos))
	}
}

		prometheus.CounterOpts{
			Namespace: "etcd",
			Subsystem: "grpc",
			Name:      "requests_total",
			Help:      "Counter of received requests.",
		}, []string{"grpc_service", "grpc_method"})

	failedCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "etcd",
			Subsystem: "grpc",
			Name:      "requests_failed_total",
			Help:      "Counter of failed requests.",
		}, []string{"grpc_service", "grpc_method", "grpc_code"})

	handlingDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "etcd",
			Subsystem: "grpc",
			Name:      "unary_requests_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) of handled unary (non-stream) requests.",
			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 13),
		}, []string{"grpc_service", "grpc_method"})
)

func init() {
	prometheus.MustRegister(receivedCounter)
	prometheus.MustRegister(failedCounter)
	prometheus.MustRegister(handlingDuration)
}

			Name:      "cmds_total",
			Help:      "Counter of cmds.",
		}, []string{"type"})

	cmdFailedCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "pd_client",
			Subsystem: "cmd",
			Name:      "cmds_failed_total",
			Help:      "Counter of failed cmds.",
		}, []string{"type"})

	cmdDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "pd_client",
			Subsystem: "cmd",
			Name:      "handle_cmds_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) of handled success cmds.",
			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 13),
		}, []string{"type"})

	cmdFailedDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "pd_client",
			Subsystem: "cmd",
			Name:      "handle_failed_cmds_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) of failed handled cmds.",
			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 13),
		}, []string{"type"})

	requestDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{

		Namespace: "doorman",
		Subsystem: "client",
		Name:      "requests",
		Help:      "Requests sent to a Doorman service.",
	}, requestLabels)

	requestErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "doorman",
		Subsystem: "client",
		Name:      "request_errors",
		Help:      "Requests sent to a Doorman service that returned an error.",
	}, requestLabels)

	requestDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "doorman",
		Subsystem: "client",
		Name:      "request_durations",
		Help:      "Duration of different requests in seconds.",
	}, requestLabels)
)

func init() {
	prometheus.MustRegister(requests)
	prometheus.MustRegister(requestErrors)
	prometheus.MustRegister(requestDurations)
}

// NOTE: We're wrapping connection package's functions and types here in the client,
// because we do not want our users to be aware of the internal connection package.

// Option configures the client's connection parameters.
type Option connection.Option

	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// TODO: create a separate histogram for recording
	// snapshot sending metric. snapshot can be large and
	// take a long time to send. So it needs a different
	// time range than other type of messages.
	msgSentDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "etcd_debugging",
			Subsystem: "rafthttp",
			Name:      "message_sent_latency_seconds",
			Help:      "message sent latency distributions.",
			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 13),
		},
		[]string{"sendingType", "remoteID", "msgType"},
	)

	msgSentFailed = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "etcd_debugging",
		Subsystem: "rafthttp",
		Name:      "message_sent_failed_total",
		Help:      "The total number of failed messages sent.",
	},
		[]string{"sendingType", "remoteID", "msgType"},
	)
)

package lucky

import (
	"github.com/prometheus/client_golang/prometheus"
)

var (
	RequestsHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "lucky_requests_ms",
		Help:    "Lucky requests",
		Buckets: prometheus.ExponentialBuckets(1, 5, 6),
	},
		[]string{"backend", "method"})
	BackendsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "lucky_backends",
		Help: "Lucky backends",
	},
		[]string{"backend"})

	FrontendsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "lucky_frontends",
		Help: "Lucky frontends",
	},
		[]string{"frontend", "type"})
)

func init() {
	prometheus.MustRegister(RequestsHistogram)
	prometheus.MustRegister(BackendsGauge)
	prometheus.MustRegister(FrontendsGauge)
}

func (mc *MetricCollection) NewHistogramVec(opts promm.HistogramOpts, labelNames []string) *promm.HistogramVec {
	c := promm.NewHistogramVec(opts, labelNames)
	mc.Add(c)
	return c
}

var (
	storeLabels = []string{"error", "operation"}
	countLabels = []string{
		"service",
		"job",
		"env",
		"product",
		"zone",
		"operation",
	}

	dnsDurations = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: namespace,
			Subsystem: "dns",
			Name:      "request_duration_seconds",
			Help:      "DNS request latencies in seconds.",
			Buckets:   []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1},
		},
		[]string{"protocol", "qtype", "rcode"},
	)
	storeCounts = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "consul",
			Name:      "responses",
			Help:      "Consul API responses.",
		},
		countLabels,
	)
	storeDurations = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{

		Help:      "The total number of bytes sent to peers.",
	},
		[]string{"To"},
	)

	receivedBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "etcd",
		Subsystem: "network",
		Name:      "peer_received_bytes_total",
		Help:      "The total number of bytes received from peers.",
	},
		[]string{"From"},
	)

	rtts = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "etcd",
		Subsystem: "network",
		Name:      "peer_round_trip_time_seconds",
		Help:      "Round-Trip-Time histogram between peers.",
		Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 14),
	},
		[]string{"To"},
	)
)

func init() {
	prometheus.MustRegister(sentBytes)
	prometheus.MustRegister(receivedBytes)
	prometheus.MustRegister(rtts)
}

var (
	// TODO(a-robinson): Add unit tests for the handling of these metrics once
	// the upstream library supports it.
	requestCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "apiserver_request_count",
			Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response code.",
		},
		[]string{"verb", "resource", "client", "code"},
	)
	requestLatencies = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "apiserver_request_latencies",
			Help: "Response latency distribution in microseconds for each verb, resource and client.",
			// Use buckets ranging from 125 ms to 8 seconds.
			Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
		},
		[]string{"verb", "resource"},
	)
	requestLatenciesSummary = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name: "apiserver_request_latencies_summary",
			Help: "Response latency summary in microseconds for each verb and resource.",
		},
		[]string{"verb", "resource"},
	)
)

// Register all metrics.
func Register() {