func TestSensorRecordSummary(t *testing.T) {
	testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer testServer.Close()

	sensor := &Sensor{
		Type: "summary",
		collector: prometheus.NewSummary(prometheus.SummaryOpts{
			Namespace: "telemetry",
			Subsystem: "sensors",
			Name:      "TestSensorRecordSummary",
			Help:      "help",
		})}
	prometheus.MustRegister(sensor.collector)
	patt := `telemetry_sensors_TestSensorRecordSummary{quantile="([\.0-9]*)"} ([0-9\.]*)`

	// need a bunch of metrics to make quantiles make any sense
	for i := 1; i <= 10; i++ {
		sensor.record(fmt.Sprintf("%v", i))
	}
	resp := getFromTestServer(t, testServer)
	expected := [][]string{{"0.5", "5"}, {"0.9", "9"}, {"0.99", "9"}}
	if !checkBuckets(resp, patt, expected) {
		t.Fatalf("Failed to get match for sensor in response")
	}

	for i := 1; i <= 5; i++ {
		// add a new record for each one in the bottom half
		sensor.record(fmt.Sprintf("%v", i))
	}
	resp = getFromTestServer(t, testServer)
	expected = [][]string{{"0.5", "4"}, {"0.9", "8"}, {"0.99", "9"}}
	if !checkBuckets(resp, patt, expected) {
		t.Fatalf("Failed to get match for sensor in response")
	}
}

func TestSensorRecordHistogram(t *testing.T) {
	testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer testServer.Close()

	sensor := &Sensor{
		Type: "histogram",
		collector: prometheus.NewHistogram(prometheus.HistogramOpts{
			Namespace: "telemetry",
			Subsystem: "sensors",
			Name:      "TestSensorRecordHistogram",
			Help:      "help",
		})}
	prometheus.MustRegister(sensor.collector)
	patt := `telemetry_sensors_TestSensorRecordHistogram_bucket{le="([\.0-9|\+Inf]*)"} ([1-9])`

	sensor.record("1.2")
	resp := getFromTestServer(t, testServer)
	expected := [][]string{{"2.5", "1"}, {"5", "1"}, {"10", "1"}, {"+Inf", "1"}}
	if !checkBuckets(resp, patt, expected) {
		t.Fatalf("Failed to get match for sensor in response")
	}
	sensor.record("1.2") // same value should add
	resp = getFromTestServer(t, testServer)
	expected = [][]string{{"2.5", "2"}, {"5", "2"}, {"10", "2"}, {"+Inf", "2"}}
	if !checkBuckets(resp, patt, expected) {
		t.Fatalf("Failed to get match for sensor in response")
	}
	sensor.record("4.5") // overlapping should overlap
	resp = getFromTestServer(t, testServer)
	expected = [][]string{{"2.5", "2"}, {"5", "3"}, {"10", "3"}, {"+Inf", "3"}}
	if !checkBuckets(resp, patt, expected) {
		t.Fatalf("Failed to get match for sensor in response")
	}
}

func TestCounter(t *testing.T) {
	s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	defer s.Close()

	scrape := func() string {
		resp, _ := http.Get(s.URL)
		buf, _ := ioutil.ReadAll(resp.Body)
		return string(buf)
	}

	namespace, subsystem, name := "ns", "ss", "foo"
	re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{alpha="alpha-value",beta="beta-value"} ([0-9\.]+)`)

	counter := NewCounterFrom(stdprometheus.CounterOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      name,
		Help:      "This is the help string.",
	}, []string{"alpha", "beta"}).With("beta", "beta-value", "alpha", "alpha-value") // order shouldn't matter

	value := func() float64 {
		matches := re.FindStringSubmatch(scrape())
		f, _ := strconv.ParseFloat(matches[1], 64)
		return f
	}

	if err := teststat.TestCounter(counter, value); err != nil {
		t.Fatal(err)
	}
}

func (s *Server) route() {
	s.r = mux.NewRouter().StrictSlash(true)
	s.r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		s.serveErrorPage("page not found", http.StatusNotFound, w, r)
	})

	s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe)
	s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt)
	s.r.Path("/metrics").Handler(
		prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler()))

	s.r.PathPrefix("/static/").Handler(
		prometheus.InstrumentHandler("static", http.HandlerFunc(s.handleStatic)))

	s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic))

	s.r.PathPrefix("/about").Handler(
		prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic)))

	s.r.HandleFunc("/room/{prefix:(pm:)?}{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom))
	s.r.Handle(
		"/room/{prefix:(pm:)?}{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic))

	s.r.Handle(
		"/prefs/reset-password",
		prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handlePrefsResetPassword))
	s.r.Handle(
		"/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify))
}

func Start(config *Config) error {
	if err := clone(config); err != nil {
		return err
	}
	handler := handler(config)

	ops := http.NewServeMux()
	if config.AllowHooks {
		ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config))))
	}
	/*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL)
		io.Copy(os.Stdout, r.Body)
	}))*/
	ops.Handle("/metrics", prometheus.UninstrumentedHandler())
	healthz.InstallHandler(ops)

	mux := http.NewServeMux()
	mux.Handle("/", prometheus.InstrumentHandler("git", handler))
	mux.Handle("/_/", http.StripPrefix("/_", ops))

	log.Printf("Serving %s on %s", config.Home, config.Listen)
	return http.ListenAndServe(config.Listen, mux)
}

func (s *Server) route() {
	s.r = mux.NewRouter().StrictSlash(true)
	s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe)
	s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt)
	s.r.Path("/metrics").Handler(
		prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler()))

	s.r.PathPrefix("/static/").Handler(
		prometheus.InstrumentHandler("static", http.StripPrefix("/static", http.HandlerFunc(s.handleStatic))))

	s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic))

	s.r.PathPrefix("/about").Handler(
		prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic)))

	s.r.HandleFunc("/room/{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom))
	s.r.Handle(
		"/room/{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic))

	s.r.Handle(
		"/prefs/reset-password",
		prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handleResetPassword))
	s.r.Handle(
		"/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify))
}

func TestGauge(t *testing.T) {
	s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	defer s.Close()

	scrape := func() string {
		resp, _ := http.Get(s.URL)
		buf, _ := ioutil.ReadAll(resp.Body)
		return string(buf)
	}

	namespace, subsystem, name := "aaa", "bbb", "ccc"
	re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{foo="bar"} ([0-9\.]+)`)

	gauge := NewGaugeFrom(stdprometheus.GaugeOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      name,
		Help:      "This is a different help string.",
	}, []string{"foo"}).With("foo", "bar")

	value := func() float64 {
		matches := re.FindStringSubmatch(scrape())
		f, _ := strconv.ParseFloat(matches[1], 64)
		return f
	}

	if err := teststat.TestGauge(gauge, value); err != nil {
		t.Fatal(err)
	}
}

func TestSensorRecordGauge(t *testing.T) {
	testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer testServer.Close()

	sensor := &Sensor{
		Type: "gauge",
		collector: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "telemetry",
			Subsystem: "sensors",
			Name:      "TestSensorRecordGauge",
			Help:      "help",
		})}

	prometheus.MustRegister(sensor.collector)
	sensor.record("1.2")
	resp := getFromTestServer(t, testServer)
	if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 1.2") != 1 {
		t.Fatalf("Failed to get match for sensor in response: %s", resp)
	}
	sensor.record("2.3")
	resp = getFromTestServer(t, testServer)
	if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 2.3") != 1 {
		t.Fatalf("Failed to get match for sensor in response: %s", resp)
	}
}

func (o *ObserveOptions) dumpMetrics() {
	if !o.printMetricsOnExit {
		return
	}
	w := httptest.NewRecorder()
	prometheus.UninstrumentedHandler().ServeHTTP(w, &http.Request{})
	if w.Code == http.StatusOK {
		fmt.Fprintf(o.out, w.Body.String())
	}
}

func scrapePrometheus(t *testing.T) string {
	server := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	defer server.Close()

	resp, err := http.Get(server.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()

	buf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}

	return strings.TrimSpace(string(buf))
}

func ExampleInstrumentHandler() {
	// Handle the "/doc" endpoint with the standard http.FileServer handler.
	// By wrapping the handler with InstrumentHandler, request count,
	// request and response sizes, and request latency are automatically
	// exported to Prometheus, partitioned by HTTP status code and method
	// and by the handler name (here "fileserver").
	http.Handle("/doc", prometheus.InstrumentHandler(
		"fileserver", http.FileServer(http.Dir("/usr/share/doc")),
	))
	// The Prometheus handler still has to be registered to handle the
	// "/metrics" endpoint. The handler returned by prometheus.Handler() is
	// already instrumented - with "prometheus" as the handler name. In this
	// example, we want the handler name to be "metrics", so we instrument
	// the uninstrumented Prometheus handler ourselves.
	http.Handle("/metrics", prometheus.InstrumentHandler(
		"metrics", prometheus.UninstrumentedHandler(),
	))
}

// partial metric name parses ok and write out as expected
func TestSensorPartialName(t *testing.T) {

	testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer testServer.Close()

	jsonFragment := []byte(`[{
	"name": "telemetry_sensors_partial_name",
	"help": "help text",
	"type": "counter"}]`)
	sensor := parseSensors(t, jsonFragment)[0]
	if _, ok := sensor.collector.(prometheus.Counter); !ok {
		t.Fatalf("Incorrect collector; expected Counter but got %v", sensor.collector)
	}

	sensor.record("1")
	resp := getFromTestServer(t, testServer)
	if strings.Count(resp, "telemetry_sensors_partial_name 1") != 1 {
		t.Fatalf("Failed to get match for sensor in response: %s", resp)
	}
}

func TestSensorPollAction(t *testing.T) {
	testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer testServer.Close()
	cmd, _ := commands.NewCommand("./testdata/test.sh measureStuff", "0")
	sensor := &Sensor{
		Type:     "counter",
		checkCmd: cmd,
		collector: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: "telemetry",
			Subsystem: "sensors",
			Name:      "TestSensorPollAction",
			Help:      "help",
		})}
	prometheus.MustRegister(sensor.collector)
	sensor.PollAction()
	resp := getFromTestServer(t, testServer)
	if strings.Count(resp, "telemetry_sensors_TestSensorPollAction 42") != 1 {
		t.Fatalf("Failed to get match for sensor in response: %s", resp)
	}
}

func export(json string) ([]byte, error) {
	exporter := NewExporter(&testScraper{json})
	prometheus.MustRegister(exporter)
	defer prometheus.Unregister(exporter)

	server := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer server.Close()

	response, err := http.Get(server.URL)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
}

func TestSummary(t *testing.T) {
	s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	defer s.Close()

	scrape := func() string {
		resp, _ := http.Get(s.URL)
		buf, _ := ioutil.ReadAll(resp.Body)
		return string(buf)
	}

	namespace, subsystem, name := "test", "prometheus", "summary"
	re50 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.5"} ([0-9\.]+)`)
	re90 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.9"} ([0-9\.]+)`)
	re99 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.99"} ([0-9\.]+)`)

	summary := NewSummaryFrom(stdprometheus.SummaryOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      name,
		Help:      "This is the help string for the summary.",
	}, []string{"a", "b"}).With("b", "b").With("a", "a")

	quantiles := func() (float64, float64, float64, float64) {
		buf := scrape()
		match50 := re50.FindStringSubmatch(buf)
		p50, _ := strconv.ParseFloat(match50[1], 64)
		match90 := re90.FindStringSubmatch(buf)
		p90, _ := strconv.ParseFloat(match90[1], 64)
		match99 := re99.FindStringSubmatch(buf)
		p99, _ := strconv.ParseFloat(match99[1], 64)
		p95 := p90 + ((p99 - p90) / 2) // Prometheus, y u no p95??? :< #yolo
		return p50, p90, p95, p99
	}

	if err := teststat.TestHistogram(summary, quantiles, 0.01); err != nil {
		t.Fatal(err)
	}
}

// 1. parse args
// 2. start the prometheus listener if configured
// 3. start the loghisto metric system
// 4. start the processing and printing goroutines
// 5. open the pcap handler
// 6. hand off packets from the handler to the decoder
func main() {
	portsArg := flag.String("ports", "4001,2379", "etcd listening ports")
	iface := flag.String("iface", "eth0", "interface for sniffing traffic on")
	promisc := flag.Bool("promiscuous", false, "promiscuous mode")
	period := flag.Uint("period", 60, "seconds between submissions")
	topK := flag.Uint("topk", 10, "submit stats for the top <K> sniffed paths")
	prometheusPort := flag.Uint("prometheus-port", 0, "port for prometheus exporter to listen on")
	flag.Parse()

	if *prometheusPort != 0 {
		http.Handle("/metrics", prometheus.UninstrumentedHandler())
		go http.ListenAndServe(":"+strconv.Itoa(int(*prometheusPort)), nil)
	}

	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	ms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false)
	ms.Start()
	metricStream := make(chan *loghisto.ProcessedMetricSet, 2)
	ms.SubscribeToProcessedMetrics(metricStream)
	defer ms.UnsubscribeFromProcessedMetrics(metricStream)

	go statPrinter(metricStream, *topK, *period)

	ports := []uint16{}
	for _, p := range strings.Split(*portsArg, ",") {
		p, err := strconv.Atoi(p)
		if err == nil {
			ports = append(ports, uint16(p))
		}
	}

	h, err := pcap.Openlive(*iface, 1518, *promisc, 1000)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer h.Close()

	portArray := strings.Split(*portsArg, ",")
	dst := strings.Join(portArray, " or dst port ")
	src := strings.Join(portArray, " or src port ")
	filter := fmt.Sprintf("tcp and (dst port %s or src port %s)", dst, src)
	fmt.Println("using bpf filter: ", filter)
	if err := h.Setfilter(filter); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	unparsedPackets := make(chan *pcap.Packet, 10240)
	parsedPackets := make(chan *pcap.Packet, 10240)
	for i := 0; i < 5; i++ {
		go packetDecoder(unparsedPackets, parsedPackets)
	}

	processors := []chan *pcap.Packet{}
	for i := 0; i < 50; i++ {
		p := make(chan *pcap.Packet, 10240)
		processors = append(processors, p)
		go processor(ms, p)
	}

	go streamRouter(ports, parsedPackets, processors)

	for {
		pkt := h.Next()
		if pkt != nil {
			select {
			case unparsedPackets <- pkt:
			default:
				fmt.Println("SHEDDING IN MAIN")
			}
		}
	}
}

func prometheusHandler() gin.HandlerFunc {
	h := prometheus.UninstrumentedHandler()
	return func(c *gin.Context) {
		h.ServeHTTP(c.Writer, c.Request)
	}
}

func TestHistogram(t *testing.T) {
	// Prometheus reports histograms as a count of observations that fell into
	// each predefined bucket, with the bucket value representing a global upper
	// limit. That is, the count monotonically increases over the buckets. This
	// requires a different strategy to test.

	s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	defer s.Close()

	scrape := func() string {
		resp, _ := http.Get(s.URL)
		buf, _ := ioutil.ReadAll(resp.Body)
		return string(buf)
	}

	namespace, subsystem, name := "test", "prometheus", "histogram"
	re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `_bucket{x="1",le="([0-9]+|\+Inf)"} ([0-9\.]+)`)

	numStdev := 3
	bucketMin := (teststat.Mean - (numStdev * teststat.Stdev))
	bucketMax := (teststat.Mean + (numStdev * teststat.Stdev))
	if bucketMin < 0 {
		bucketMin = 0
	}
	bucketCount := 10
	bucketDelta := (bucketMax - bucketMin) / bucketCount
	buckets := []float64{}
	for i := bucketMin; i <= bucketMax; i += bucketDelta {
		buckets = append(buckets, float64(i))
	}

	histogram := NewHistogramFrom(stdprometheus.HistogramOpts{
		Namespace: namespace,
		Subsystem: subsystem,
		Name:      name,
		Help:      "This is the help string for the histogram.",
		Buckets:   buckets,
	}, []string{"x"}).With("x", "1")

	// Can't TestHistogram, because Prometheus Histograms don't dynamically
	// compute quantiles. Instead, they fill up buckets. So, let's populate the
	// histogram kind of manually.
	teststat.PopulateNormalHistogram(histogram, rand.Int())

	// Then, we use ExpectedObservationsLessThan to validate.
	for _, line := range strings.Split(scrape(), "\n") {
		match := re.FindStringSubmatch(line)
		if match == nil {
			continue
		}

		bucket, _ := strconv.ParseInt(match[1], 10, 64)
		have, _ := strconv.ParseInt(match[2], 10, 64)

		want := teststat.ExpectedObservationsLessThan(bucket)
		if match[1] == "+Inf" {
			want = int64(teststat.Count) // special case
		}

		// Unfortunately, we observe experimentally that Prometheus is quite
		// imprecise at the extremes. I'm setting a very high tolerance for now.
		// It would be great to dig in and figure out whether that's a problem
		// with my Expected calculation, or in Prometheus.
		tolerance := 0.25
		if delta := math.Abs(float64(want) - float64(have)); (delta / float64(want)) > tolerance {
			t.Errorf("Bucket %d: want %d, have %d (%.1f%%)", bucket, want, have, (100.0 * delta / float64(want)))
		}
	}
}