func TestPrometheusCollector(t *testing.T) {
	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string {
		return map[string]string{
			"zone.name": "hello",
		}
	})
	prometheus.MustRegister(c)
	defer prometheus.Unregister(c)

	rw := httptest.NewRecorder()
	prometheus.Handler().ServeHTTP(rw, &http.Request{})

	metricsFile := "testdata/prometheus_metrics"
	wantMetrics, err := ioutil.ReadFile(metricsFile)
	if err != nil {
		t.Fatalf("unable to read input test file %s", metricsFile)
	}

	wantLines := strings.Split(string(wantMetrics), "\n")
	gotLines := strings.Split(string(rw.Body.String()), "\n")

	// Until the Prometheus Go client library offers better testability
	// (https://github.com/prometheus/client_golang/issues/58), we simply compare
	// verbatim text-format metrics outputs, but ignore certain metric lines
	// whose value depends on the current time or local circumstances.
	for i, want := range wantLines {
		if !includeRe.MatchString(want) || ignoreRe.MatchString(want) {
			continue
		}
		if want != gotLines[i] {
			t.Fatalf("want %s, got %s", want, gotLines[i])
		}
	}
}

func TestServer(t *testing.T) {
	ts := int(time.Now().Unix())
	storageToTest := []string{
		fmt.Sprintf("local:./test-%d.db", ts),
		fmt.Sprintf("rethinkdb:localhost:28015/annotst%d", ts),
	}

	for _, storage := range storageToTest {
		log.Printf("testing storage: %s", storage)

		s := NewSetup(t, storage)

		s.testAddAndQuery()
		s.testDefaultValues()
		s.testTagStats()
		s.testBrokenJSON()
		s.testMetrics()
		s.testAllTags()
		s.testAll()

		s.Server.Close()
		s.Ctx.storage.Cleanup()
		prometheus.Unregister(s.Ctx)
	}
}

func (reg *MetricRegistry) Unregister(name string) {
	if metric := reg.metrics[name]; metric != nil {
		log.Infof("metric unregistered;name:<%s>", name)
		prometheus.Unregister(metric)
		delete(reg.metrics, name)
	}
}

// NewSensors creates new sensors from a raw config
func NewSensors(raw []interface{}) ([]*Sensor, error) {
	var sensors []*Sensor
	if err := utils.DecodeRaw(raw, &sensors); err != nil {
		return nil, fmt.Errorf("Sensor configuration error: %v", err)
	}
	for _, s := range sensors {
		check, err := commands.NewCommand(s.CheckExec, s.Timeout)
		if err != nil {
			return nil, fmt.Errorf("could not parse check in sensor %s: %s", s.Name, err)
		}
		check.Name = fmt.Sprintf("%s.sensor", s.Name)
		s.checkCmd = check

		// the prometheus client lib's API here is baffling... they don't expose
		// an interface or embed their Opts type in each of the Opts "subtypes",
		// so we can't share the initialization.
		switch {
		case s.Type == "counter":
			s.collector = prometheus.NewCounter(prometheus.CounterOpts{
				Namespace: s.Namespace,
				Subsystem: s.Subsystem,
				Name:      s.Name,
				Help:      s.Help,
			})
		case s.Type == "gauge":
			s.collector = prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: s.Namespace,
				Subsystem: s.Subsystem,
				Name:      s.Name,
				Help:      s.Help,
			})
		case s.Type == "histogram":
			s.collector = prometheus.NewHistogram(prometheus.HistogramOpts{
				Namespace: s.Namespace,
				Subsystem: s.Subsystem,
				Name:      s.Name,
				Help:      s.Help,
			})
		case s.Type == "summary":
			s.collector = prometheus.NewSummary(prometheus.SummaryOpts{
				Namespace: s.Namespace,
				Subsystem: s.Subsystem,
				Name:      s.Name,
				Help:      s.Help,
			})
		default:
			return nil, fmt.Errorf("invalid sensor type: %s", s.Type)
		}
		// we're going to unregister before every attempt to register
		// so that we can reload config
		prometheus.Unregister(s.collector)
		if err := prometheus.Register(s.collector); err != nil {
			return nil, err
		}
	}
	return sensors, nil
}

func TestPrometheusCollector(t *testing.T) {
	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(container *info.ContainerInfo) map[string]string {
		s := DefaultContainerLabels(container)
		s["zone.name"] = "hello"
		return s
	})
	prometheus.MustRegister(c)
	defer prometheus.Unregister(c)

	testPrometheusCollector(t, c, "testdata/prometheus_metrics")
}

func TestPrometheusCollector(t *testing.T) {
	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string {
		return map[string]string{
			"zone.name": "hello",
		}
	})
	prometheus.MustRegister(c)
	defer prometheus.Unregister(c)

	testPrometheusCollector(t, c, "testdata/prometheus_metrics")
}

func (h *handler) Stop() {
	if h.advertiser != nil {
		h.advertiser.stop()
	}

	if listener := h.listener; listener != nil {
		h.listener = nil
		if err := listener.Close(); err != nil {
			h.errorSink.Post(err)
		}
	}

	for _, c := range h.collectors() {
		prom.Unregister(c)
	}
}

func TestPrometheusCollector_scrapeFailure(t *testing.T) {
	provider := &erroringSubcontainersInfoProvider{
		successfulProvider: testSubcontainersInfoProvider{},
		shouldFail:         true,
	}

	c := NewPrometheusCollector(provider, func(name string) map[string]string {
		return map[string]string{
			"zone.name": "hello",
		}
	})
	prometheus.MustRegister(c)
	defer prometheus.Unregister(c)

	testPrometheusCollector(t, c, "testdata/prometheus_metrics_failure")

	provider.shouldFail = false

	testPrometheusCollector(t, c, "testdata/prometheus_metrics")
}

func TestPrometheusCollector_scrapeFailure(t *testing.T) {
	provider := &erroringSubcontainersInfoProvider{
		successfulProvider: testSubcontainersInfoProvider{},
		shouldFail:         true,
	}

	c := NewPrometheusCollector(provider, func(container *info.ContainerInfo) map[string]string {
		s := DefaultContainerLabels(container)
		s["zone.name"] = "hello"
		return s
	})
	prometheus.MustRegister(c)
	defer prometheus.Unregister(c)

	testPrometheusCollector(t, c, "testdata/prometheus_metrics_failure")

	provider.shouldFail = false

	testPrometheusCollector(t, c, "testdata/prometheus_metrics")
}

func export(json string) ([]byte, error) {
	exporter := NewExporter(&testScraper{json})
	prometheus.MustRegister(exporter)
	defer prometheus.Unregister(exporter)

	server := httptest.NewServer(prometheus.UninstrumentedHandler())
	defer server.Close()

	response, err := http.Get(server.URL)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
}

//.........这里部分代码省略.........
				mValueBytes.Set(float64(s.ValueBytes))
				mLookups.Add(float64(s.Lookups))
				mLookupErrors.Add(float64(s.LookupErrors))
				mReads.Add(float64(s.Reads))
				mReadErrors.Add(float64(s.ReadErrors))
				mWrites.Add(float64(s.Writes))
				mWriteErrors.Add(float64(s.WriteErrors))
				mWritesOverridden.Add(float64(s.WritesOverridden))
				mDeletes.Add(float64(s.Deletes))
				mDeleteErrors.Add(float64(s.DeleteErrors))
				mDeletesOverridden.Add(float64(s.DeletesOverridden))
				mOutBulkSets.Add(float64(s.OutBulkSets))
				mOutBulkSetValues.Add(float64(s.OutBulkSetValues))
				mOutBulkSetPushes.Add(float64(s.OutBulkSetPushes))
				mOutBulkSetPushValues.Add(float64(s.OutBulkSetPushValues))
				mInBulkSets.Add(float64(s.InBulkSets))
				mInBulkSetDrops.Add(float64(s.InBulkSetDrops))
				mInBulkSetInvalids.Add(float64(s.InBulkSetInvalids))
				mInBulkSetWrites.Add(float64(s.InBulkSetWrites))
				mInBulkSetWriteErrors.Add(float64(s.InBulkSetWriteErrors))
				mInBulkSetWritesOverridden.Add(float64(s.InBulkSetWritesOverridden))
				mOutBulkSetAcks.Add(float64(s.OutBulkSetAcks))
				mInBulkSetAcks.Add(float64(s.InBulkSetAcks))
				mInBulkSetAckDrops.Add(float64(s.InBulkSetAckDrops))
				mInBulkSetAckInvalids.Add(float64(s.InBulkSetAckInvalids))
				mInBulkSetAckWrites.Add(float64(s.InBulkSetAckWrites))
				mInBulkSetAckWriteErrors.Add(float64(s.InBulkSetAckWriteErrors))
				mInBulkSetAckWritesOverridden.Add(float64(s.InBulkSetAckWritesOverridden))
				mOutPullReplications.Add(float64(s.OutPullReplications))
				mOutPullReplicationSeconds.Set(float64(s.OutPullReplicationNanoseconds) / 1000000000)
				mInPullReplications.Add(float64(s.InPullReplications))
				mInPullReplicationDrops.Add(float64(s.InPullReplicationDrops))
				mInPullReplicationInvalids.Add(float64(s.InPullReplicationInvalids))
				mExpiredDeletions.Add(float64(s.ExpiredDeletions))
				mCompactions.Add(float64(s.Compactions))
				mSmallFileCompactions.Add(float64(s.SmallFileCompactions))
			} else {
				log.Printf("%s\n", stats)
			}
		}
		prometheus.Unregister(mRingChanges)
		prometheus.Unregister(mRingChangeCloses)
		prometheus.Unregister(mMsgToNodes)
		prometheus.Unregister(mMsgToNodeNoRings)
		prometheus.Unregister(mMsgToNodeNoNodes)
		prometheus.Unregister(mMsgToOtherReplicas)
		prometheus.Unregister(mMsgToOtherReplicasNoRings)
		prometheus.Unregister(mListenErrors)
		prometheus.Unregister(mIncomingConnections)
		prometheus.Unregister(mDials)
		prometheus.Unregister(mDialErrors)
		prometheus.Unregister(mOutgoingConnections)
		prometheus.Unregister(mMsgChanCreations)
		prometheus.Unregister(mMsgToAddrs)
		prometheus.Unregister(mMsgToAddrQueues)
		prometheus.Unregister(mMsgToAddrTimeoutDrops)
		prometheus.Unregister(mMsgToAddrShutdownDrops)
		prometheus.Unregister(mMsgReads)
		prometheus.Unregister(mMsgReadErrors)
		prometheus.Unregister(mMsgWrites)
		prometheus.Unregister(mMsgWriteErrors)
		prometheus.Unregister(mValues)
		prometheus.Unregister(mValueBytes)
		prometheus.Unregister(mLookups)
		prometheus.Unregister(mLookupErrors)
		prometheus.Unregister(mReads)
		prometheus.Unregister(mReadErrors)
		prometheus.Unregister(mWrites)
		prometheus.Unregister(mWriteErrors)
		prometheus.Unregister(mWritesOverridden)
		prometheus.Unregister(mDeletes)
		prometheus.Unregister(mDeleteErrors)
		prometheus.Unregister(mDeletesOverridden)
		prometheus.Unregister(mOutBulkSets)
		prometheus.Unregister(mOutBulkSetValues)
		prometheus.Unregister(mOutBulkSetPushes)
		prometheus.Unregister(mOutBulkSetPushValues)
		prometheus.Unregister(mInBulkSets)
		prometheus.Unregister(mInBulkSetDrops)
		prometheus.Unregister(mInBulkSetInvalids)
		prometheus.Unregister(mInBulkSetWrites)
		prometheus.Unregister(mInBulkSetWriteErrors)
		prometheus.Unregister(mInBulkSetWritesOverridden)
		prometheus.Unregister(mOutBulkSetAcks)
		prometheus.Unregister(mInBulkSetAcks)
		prometheus.Unregister(mInBulkSetAckDrops)
		prometheus.Unregister(mInBulkSetAckInvalids)
		prometheus.Unregister(mInBulkSetAckWrites)
		prometheus.Unregister(mInBulkSetAckWriteErrors)
		prometheus.Unregister(mInBulkSetAckWritesOverridden)
		prometheus.Unregister(mOutPullReplications)
		prometheus.Unregister(mOutPullReplicationSeconds)
		prometheus.Unregister(mInPullReplications)
		prometheus.Unregister(mInPullReplicationDrops)
		prometheus.Unregister(mInPullReplicationInvalids)
		prometheus.Unregister(mExpiredDeletions)
		prometheus.Unregister(mCompactions)
		prometheus.Unregister(mSmallFileCompactions)
	}(s.msgRing)
}

//.........这里部分代码省略.........
                "addr": "10.123.2.25:6789\/0"
            },
            {
                "rank": 3,
                "name": "test-mon04",
                "addr": "10.123.2.26:6789\/0"
            },
            {
                "rank": 4,
                "name": "test-mon05",
                "addr": "10.123.2.27:6789\/0"
            }
        ]
    }
}
`,
			[]*regexp.Regexp{
				regexp.MustCompile(`ceph_monitor_avail_bytes{monitor="test-mon01"} 3.899175e`),
				regexp.MustCompile(`ceph_monitor_avail_bytes{monitor="test-mon02"} 3.89855048e`),
				regexp.MustCompile(`ceph_monitor_avail_bytes{monitor="test-mon03"} 3.89634996e`),
				regexp.MustCompile(`ceph_monitor_avail_bytes{monitor="test-mon04"} 3.90004076e`),
				regexp.MustCompile(`ceph_monitor_avail_bytes{monitor="test-mon05"} 3.89847124e`),
				regexp.MustCompile(`ceph_monitor_avail_percent{monitor="test-mon01"} 94`),
				regexp.MustCompile(`ceph_monitor_avail_percent{monitor="test-mon02"} 94`),
				regexp.MustCompile(`ceph_monitor_avail_percent{monitor="test-mon03"} 94`),
				regexp.MustCompile(`ceph_monitor_avail_percent{monitor="test-mon04"} 94`),
				regexp.MustCompile(`ceph_monitor_avail_percent{monitor="test-mon05"} 94`),
				regexp.MustCompile(`ceph_monitor_clock_skew_seconds{monitor="test-mon01"} 0`),
				regexp.MustCompile(`ceph_monitor_clock_skew_seconds{monitor="test-mon02"} -2e-06`),
				regexp.MustCompile(`ceph_monitor_clock_skew_seconds{monitor="test-mon03"} -2e-06`),
				regexp.MustCompile(`ceph_monitor_clock_skew_seconds{monitor="test-mon04"} -1.9e-05`),
				regexp.MustCompile(`ceph_monitor_clock_skew_seconds{monitor="test-mon05"} -0.000628`),
				regexp.MustCompile(`ceph_monitor_latency_seconds{monitor="test-mon01"} 0`),
				regexp.MustCompile(`ceph_monitor_latency_seconds{monitor="test-mon02"} 0.000815`),
				regexp.MustCompile(`ceph_monitor_latency_seconds{monitor="test-mon03"} 0.000829`),
				regexp.MustCompile(`ceph_monitor_latency_seconds{monitor="test-mon04"} 0.000609`),
				regexp.MustCompile(`ceph_monitor_latency_seconds{monitor="test-mon05"} 0.000659`),
				regexp.MustCompile(`ceph_monitor_quorum_count 5`),
				regexp.MustCompile(`ceph_monitor_store_log_bytes{monitor="test-mon01"} 609694`),
				regexp.MustCompile(`ceph_monitor_store_log_bytes{monitor="test-mon02"} 871605`),
				regexp.MustCompile(`ceph_monitor_store_log_bytes{monitor="test-mon03"} 871605`),
				regexp.MustCompile(`ceph_monitor_store_log_bytes{monitor="test-mon04"} 871605`),
				regexp.MustCompile(`ceph_monitor_store_log_bytes{monitor="test-mon05"} 871605`),
				regexp.MustCompile(`ceph_monitor_store_misc_bytes{monitor="test-mon01"} 1.780672385e`),
				regexp.MustCompile(`ceph_monitor_store_misc_bytes{monitor="test-mon02"} 1.843476609e`),
				regexp.MustCompile(`ceph_monitor_store_misc_bytes{monitor="test-mon03"} 2.068596982e`),
				regexp.MustCompile(`ceph_monitor_store_misc_bytes{monitor="test-mon04"} 1.691100542e`),
				regexp.MustCompile(`ceph_monitor_store_misc_bytes{monitor="test-mon05"} 1.851614337e`),
				regexp.MustCompile(`ceph_monitor_store_sst_bytes{monitor="test-mon01"} 1`),
				regexp.MustCompile(`ceph_monitor_store_sst_bytes{monitor="test-mon02"} 2`),
				regexp.MustCompile(`ceph_monitor_store_sst_bytes{monitor="test-mon03"} 3`),
				regexp.MustCompile(`ceph_monitor_store_sst_bytes{monitor="test-mon04"} 4`),
				regexp.MustCompile(`ceph_monitor_store_sst_bytes{monitor="test-mon05"} 5`),
				regexp.MustCompile(`ceph_monitor_store_capacity_bytes{monitor="test-mon01"} 1.781282079e`),
				regexp.MustCompile(`ceph_monitor_store_capacity_bytes{monitor="test-mon02"} 1.844348214e`),
				regexp.MustCompile(`ceph_monitor_store_capacity_bytes{monitor="test-mon03"} 2.069468587e`),
				regexp.MustCompile(`ceph_monitor_store_capacity_bytes{monitor="test-mon04"} 1.691972147e`),
				regexp.MustCompile(`ceph_monitor_store_capacity_bytes{monitor="test-mon05"} 1.852485942e`),
				regexp.MustCompile(`ceph_monitor_capacity_bytes{monitor="test-mon01"} 4.12718256e`),
				regexp.MustCompile(`ceph_monitor_capacity_bytes{monitor="test-mon02"} 4.12718256e`),
				regexp.MustCompile(`ceph_monitor_capacity_bytes{monitor="test-mon03"} 4.12718256e`),
				regexp.MustCompile(`ceph_monitor_capacity_bytes{monitor="test-mon04"} 4.12718256e`),
				regexp.MustCompile(`ceph_monitor_capacity_bytes{monitor="test-mon05"} 4.12718256e`),
				regexp.MustCompile(`ceph_monitor_used_bytes{monitor="test-mon01"} 1.812852e`),
				regexp.MustCompile(`ceph_monitor_used_bytes{monitor="test-mon02"} 1.875304e`),
				regexp.MustCompile(`ceph_monitor_used_bytes{monitor="test-mon03"} 2.095356e`),
				regexp.MustCompile(`ceph_monitor_used_bytes{monitor="test-mon04"} 1.726276e`),
				regexp.MustCompile(`ceph_monitor_used_bytes{monitor="test-mon05"} 1.883228e`),
			},
		},
	} {
		func() {
			collector := NewMonitorCollector(NewNoopConn(tt.input))
			if err := prometheus.Register(collector); err != nil {
				t.Fatalf("collector failed to register: %s", err)
			}
			defer prometheus.Unregister(collector)

			server := httptest.NewServer(prometheus.Handler())
			defer server.Close()

			resp, err := http.Get(server.URL)
			if err != nil {
				t.Fatalf("unexpected failed response from prometheus: %s", err)
			}
			defer resp.Body.Close()

			buf, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Fatalf("failed reading server response: %s", err)
			}

			for _, re := range tt.regexes {
				if !re.Match(buf) {
					t.Errorf("failed matching: %q", re)
				}
			}
		}()
	}
}

// Deregister removes all the metrics in the provided namespace from the
// global metrics registry
func Deregister(n *Namespace) {
	prometheus.Unregister(n)
}

func ExampleRegister() {
	// Imagine you have a worker pool and want to count the tasks completed.
	taskCounter := prometheus.NewCounter(prometheus.CounterOpts{
		Subsystem: "worker_pool",
		Name:      "completed_tasks_total",
		Help:      "Total number of tasks completed.",
	})
	// This will register fine.
	if err := prometheus.Register(taskCounter); err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("taskCounter registered.")
	}
	// Don't forget to tell the HTTP server about the Prometheus handler.
	// (In a real program, you still need to start the HTTP server...)
	http.Handle("/metrics", prometheus.Handler())

	// Now you can start workers and give every one of them a pointer to
	// taskCounter and let it increment it whenever it completes a task.
	taskCounter.Inc() // This has to happen somewhere in the worker code.

	// But wait, you want to see how individual workers perform. So you need
	// a vector of counters, with one element for each worker.
	taskCounterVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: "worker_pool",
			Name:      "completed_tasks_total",
			Help:      "Total number of tasks completed.",
		},
		[]string{"worker_id"},
	)

	// Registering will fail because we already have a metric of that name.
	if err := prometheus.Register(taskCounterVec); err != nil {
		fmt.Println("taskCounterVec not registered:", err)
	} else {
		fmt.Println("taskCounterVec registered.")
	}

	// To fix, first unregister the old taskCounter.
	if prometheus.Unregister(taskCounter) {
		fmt.Println("taskCounter unregistered.")
	}

	// Try registering taskCounterVec again.
	if err := prometheus.Register(taskCounterVec); err != nil {
		fmt.Println("taskCounterVec not registered:", err)
	} else {
		fmt.Println("taskCounterVec registered.")
	}
	// Bummer! Still doesn't work.

	// Prometheus will not allow you to ever export metrics with
	// inconsistent help strings or label names. After unregistering, the
	// unregistered metrics will cease to show up in the /metrics HTTP
	// response, but the registry still remembers that those metrics had
	// been exported before. For this example, we will now choose a
	// different name. (In a real program, you would obviously not export
	// the obsolete metric in the first place.)
	taskCounterVec = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: "worker_pool",
			Name:      "completed_tasks_by_id",
			Help:      "Total number of tasks completed.",
		},
		[]string{"worker_id"},
	)
	if err := prometheus.Register(taskCounterVec); err != nil {
		fmt.Println("taskCounterVec not registered:", err)
	} else {
		fmt.Println("taskCounterVec registered.")
	}
	// Finally it worked!

	// The workers have to tell taskCounterVec their id to increment the
	// right element in the metric vector.
	taskCounterVec.WithLabelValues("42").Inc() // Code from worker 42.

	// Each worker could also keep a reference to their own counter element
	// around. Pick the counter at initialization time of the worker.
	myCounter := taskCounterVec.WithLabelValues("42") // From worker 42 initialization code.
	myCounter.Inc()                                   // Somewhere in the code of that worker.

	// Note that something like WithLabelValues("42", "spurious arg") would
	// panic (because you have provided too many label values). If you want
	// to get an error instead, use GetMetricWithLabelValues(...) instead.
	notMyCounter, err := taskCounterVec.GetMetricWithLabelValues("42", "spurious arg")
	if err != nil {
		fmt.Println("Worker initialization failed:", err)
	}
	if notMyCounter == nil {
		fmt.Println("notMyCounter is nil.")
	}

	// A different (and somewhat tricky) approach is to use
	// ConstLabels. ConstLabels are pairs of label names and label values
	// that never change. You might ask what those labels are good for (and
	// rightfully so - if they never change, they could as well be part of
	// the metric name). There are essentially two use-cases: The first is
	// if labels are constant throughout the lifetime of a binary execution,
//.........这里部分代码省略.........

// Periodically queries a Mesos slave and updates statistics of each running task
func slavePoller(c *http.Client, conf *Config, frameworkRegistry *frameworkRegistry, slave Slave, erroredSlaves *map[string]struct{}) {
	var knownTasks map[string]taskMetric
	var monitoredTasks []MonitoredTask

	knownTasks = make(map[string]taskMetric)

	slaveStatsUrl := fmt.Sprintf("http://%s/monitor/statistics.json", slave.address())

	constLabels := prometheus.Labels{"slave_pid": slave.Pid}

	cpusLimitGauge := newGaugeVec(
		constLabels,
		"CPU limit of the task.",
		"cpus_limit",
	)

	cpusSystemTimeCounter := newCounterVec(
		constLabels,
		"Absolute CPU sytem time.",
		"cpus_system_time_seconds",
	)

	cpusUserTimeCounter := newCounterVec(
		constLabels,
		"Absolute CPU user time.",
		"cpus_user_time_seconds",
	)

	memLimitGauge := newGaugeVec(
		constLabels,
		"Maximum memory available to the task.",
		"mem_limit_bytes",
	)

	memRssGauge := newGaugeVec(
		constLabels,
		"Current Memory usage.",
		"mem_rss_bytes",
	)

	t := time.Tick(conf.MesosSlaveQueryInterval)

	for _ = range t {
		log.Debugf("Scraping slave '%s'", slave.Pid)

		availableTasks := make(map[string]struct{})

		err := retrieveStats(c, &monitoredTasks, slaveStatsUrl)
		if err != nil {
			prometheus.Unregister(cpusLimitGauge)
			prometheus.Unregister(cpusSystemTimeCounter)
			prometheus.Unregister(cpusUserTimeCounter)
			prometheus.Unregister(memLimitGauge)
			prometheus.Unregister(memRssGauge)

			log.Errorf("Error retrieving stats from slave '%s' - Stopping goroutine", slave.Pid)

			(*erroredSlaves)[slave.Pid] = struct{}{}
			return
		}

		for _, item := range monitoredTasks {
			var frameworkName string
			var taskName string

			availableTasks[item.ExecutorId] = struct{}{}

			cpusLimit := item.Statistics.CpusLimit
			cpusSystemTime := item.Statistics.CpusSystemTimeSecs
			cpusUserTime := item.Statistics.CpusUserTimeSecs
			memLimit := float64(item.Statistics.MemLimitBytes)
			memRss := float64(item.Statistics.MemRssBytes)

			metric, ok := knownTasks[item.ExecutorId]
			if ok {
				frameworkName = metric.frameworkName
				taskName = metric.taskName
			} else {
				framework, err := frameworkRegistry.Get(item.FrameworkId)
				if err != nil {
					log.Debugf("Framework '%s' of task '%s' not registered - not scraping", item.FrameworkId, item.ExecutorId)
					continue
				}

				frameworkName = framework.Name
				taskName = findTaskName(item.ExecutorId, framework)

				if taskName == "" {
					log.Debugf("Could not find name of task of executor '%s' - skipping", item.ExecutorId)
					continue
				}

				log.Debugf("Found new task '%s'", item.ExecutorId)

				knownTasks[item.ExecutorId] = taskMetric{
					frameworkName: frameworkName,
					taskName:      taskName,
				}
			}
//.........这里部分代码省略.........

//.........这里部分代码省略.........
				regexp.MustCompile(`pool_used_bytes{pool="rbd"} 20`),
				regexp.MustCompile(`pool_objects_total{pool="rbd"} 0`),
				regexp.MustCompile(`pool_read_total{pool="rbd"} 4`),
				regexp.MustCompile(`pool_write_total{pool="rbd"} 6`),
			},
			reUnmatch: []*regexp.Regexp{},
		},
		{
			input: `
{"pools": [
	{"name": "rbd", "id": 11, "stats": {"bytes_used": 20, "objects": 5, "wr": 6}}
]}`,
			reMatch: []*regexp.Regexp{
				regexp.MustCompile(`pool_used_bytes{pool="rbd"} 20`),
				regexp.MustCompile(`pool_objects_total{pool="rbd"} 5`),
				regexp.MustCompile(`pool_read_total{pool="rbd"} 0`),
				regexp.MustCompile(`pool_write_total{pool="rbd"} 6`),
			},
			reUnmatch: []*regexp.Regexp{},
		},
		{
			input: `
{"pools": [
	{"name": "rbd", "id": 11, "stats": {"bytes_used": 20, "objects": 5, "rd": 4}}
]}`,
			reMatch: []*regexp.Regexp{
				regexp.MustCompile(`pool_used_bytes{pool="rbd"} 20`),
				regexp.MustCompile(`pool_objects_total{pool="rbd"} 5`),
				regexp.MustCompile(`pool_read_total{pool="rbd"} 4`),
				regexp.MustCompile(`pool_write_total{pool="rbd"} 0`),
			},
			reUnmatch: []*regexp.Regexp{},
		},
		{
			input: `
{"pools": [
    {{{{"name": "rbd", "id": 11, "stats": {"bytes_used": 20, "objects": 5, "rd": 4, "wr": 6}}
]}`,
			reMatch: []*regexp.Regexp{},
			reUnmatch: []*regexp.Regexp{
				regexp.MustCompile(`pool_used_bytes`),
				regexp.MustCompile(`pool_objects_total`),
				regexp.MustCompile(`pool_read_total`),
				regexp.MustCompile(`pool_write_total`),
			},
		},
		{
			input: `
{"pools": [
	{"name": "rbd", "id": 11, "stats": {"bytes_used": 20, "objects": 5, "rd": 4, "wr": 6}},
	{"name": "rbd-new", "id": 12, "stats": {"bytes_used": 50, "objects": 20, "rd": 10, "wr": 30}}
]}`,
			reMatch: []*regexp.Regexp{
				regexp.MustCompile(`pool_used_bytes{pool="rbd"} 20`),
				regexp.MustCompile(`pool_objects_total{pool="rbd"} 5`),
				regexp.MustCompile(`pool_read_total{pool="rbd"} 4`),
				regexp.MustCompile(`pool_write_total{pool="rbd"} 6`),
				regexp.MustCompile(`pool_used_bytes{pool="rbd-new"} 50`),
				regexp.MustCompile(`pool_objects_total{pool="rbd-new"} 20`),
				regexp.MustCompile(`pool_read_total{pool="rbd-new"} 10`),
				regexp.MustCompile(`pool_write_total{pool="rbd-new"} 30`),
			},
			reUnmatch: []*regexp.Regexp{},
		},
	} {
		func() {
			collector := NewPoolUsageCollector(NewNoopConn(tt.input))
			if err := prometheus.Register(collector); err != nil {
				t.Fatalf("collector failed to register: %s", err)
			}
			defer prometheus.Unregister(collector)

			server := httptest.NewServer(prometheus.Handler())
			defer server.Close()

			resp, err := http.Get(server.URL)
			if err != nil {
				t.Fatalf("unexpected failed response from prometheus: %s", err)
			}
			defer resp.Body.Close()

			buf, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Fatalf("failed reading server response: %s", err)
			}

			for _, re := range tt.reMatch {
				if !re.Match(buf) {
					t.Errorf("failed matching: %q", re)
				}
			}

			for _, re := range tt.reUnmatch {
				if re.Match(buf) {
					t.Errorf("should not have matched: %q", re)
				}
			}
		}()
	}
}

//.........这里部分代码省略.........
			`
{
	"osdmap": {
		"osdmap": {
			"num_osds": 0,
			"num_up_osds": 0,
			"num_in_osds": 0,
			"num_remapped_pgs": 0
		}
	},
	"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "8 pgs stale"}]}
}`,
			[]*regexp.Regexp{
				regexp.MustCompile(`stale_pgs 8`),
			},
		},
		{
			`
{
	"osdmap": {
		"osdmap": {
			"num_osds": 0,
			"num_up_osds": 0,
			"num_in_osds": 0,
			"num_remapped_pgs": 0
		}
	},
	"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "recovery 10/20 objects degraded"}]}
}`,
			[]*regexp.Regexp{
				regexp.MustCompile(`degraded_objects 10`),
			},
		},
		{
			`
{
	"osdmap": {
		"osdmap": {
			"num_osds": 0,
			"num_up_osds": 0,
			"num_in_osds": 0,
			"num_remapped_pgs": 0
		}
	},
	"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "3/20 in osds are down"}]}
}`,
			[]*regexp.Regexp{
				regexp.MustCompile(`osds_down 3`),
			},
		},
		{
			`
{
	"osdmap": {
		"osdmap": {
			"num_osds": 1200,
			"num_up_osds": 1200,
			"num_in_osds": 1190,
			"num_remapped_pgs": 10
		}
	},
	"health": {"summary": []}
}`,
			[]*regexp.Regexp{
				regexp.MustCompile(`osds 1200`),
				regexp.MustCompile(`osds_up 1200`),
				regexp.MustCompile(`osds_in 1190`),
				regexp.MustCompile(`pgs_remapped 10`),
			},
		},
	} {
		func() {
			collector := NewClusterHealthCollector(NewNoopConn(tt.input))
			if err := prometheus.Register(collector); err != nil {
				t.Fatalf("collector failed to register: %s", err)
			}
			defer prometheus.Unregister(collector)

			server := httptest.NewServer(prometheus.Handler())
			defer server.Close()

			resp, err := http.Get(server.URL)
			if err != nil {
				t.Fatalf("unexpected failed response from prometheus: %s", err)
			}
			defer resp.Body.Close()

			buf, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Fatalf("failed reading server response: %s", err)
			}

			for _, re := range tt.regexes {
				if !re.Match(buf) {
					t.Errorf("failed matching: %q", re)
				}
			}
		}()
	}
}

func unregisterMetrics() {
	prometheus.Unregister(timeouts)
	prometheus.Unregister(errors)
	prometheus.Unregister(requestSum)
	prometheus.Unregister(requestSuccess)
	prometheus.Unregister(requestDuration)
	prometheus.Unregister(connOpen)
	prometheus.Unregister(connError)
	prometheus.Unregister(bytesWritten)
	prometheus.Unregister(bytesRead)
	prometheus.Unregister(writeError)
	prometheus.Unregister(readError)
	prometheus.Unregister(statusCodes)
	prometheus.Unregister(errorMessages)
}

//.........这里部分代码省略.........
			},
		},
		{
			input: `
{
	"osdmap": {
		"osdmap": {
			"num_osds": 1200,
			"num_up_osds": 1200,
			"num_in_osds": 1190,
			"num_remapped_pgs": 10
		}
	},
	"health": { "overall_status": "HEALTH_WARN" } }`,
			regexes: []*regexp.Regexp{
				regexp.MustCompile(`health_status 1`),
			},
		},
		{
			input: `
{
	"osdmap": {
		"osdmap": {
			"num_osds": 1200,
			"num_up_osds": 1200,
			"num_in_osds": 1190,
			"num_remapped_pgs": 10
		}
	},
	"health": { "overall_status": "HEALTH_ERR" } }`,
			regexes: []*regexp.Regexp{
				regexp.MustCompile(`health_status 2`),
			},
		},
		{
			input: `
$ sudo ceph -s
    cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37
     health HEALTH_OK
     monmap e3: 3 mons at {mon01,mon02,mon03}
  recovery io 5779 MB/s, 4 keys/s, 1522 objects/s
  client io 4273 kB/s rd, 2740 MB/s wr, 2863 op/s
`,
			regexes: []*regexp.Regexp{
				regexp.MustCompile(`recovery_io_bytes 5.779e`),
				regexp.MustCompile(`recovery_io_keys 4`),
				regexp.MustCompile(`recovery_io_objects 1522`),
				regexp.MustCompile(`client_io_ops 2863`),
				regexp.MustCompile(`client_io_read_bytes 4.273e`),
				regexp.MustCompile(`client_io_write_bytes 2.74e`),
			},
		},
		{
			input: `
$ sudo ceph -s
    cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37
     health HEALTH_OK
     monmap e3: 3 mons at {mon01,mon02,mon03}
  recovery io 5779 MB/s, 4 keys/s, 1522 objects/s
  client io 2863 op/s rd, 5847 op/s wr
`,
			regexes: []*regexp.Regexp{
				regexp.MustCompile(`recovery_io_bytes 5.779e`),
				regexp.MustCompile(`recovery_io_keys 4`),
				regexp.MustCompile(`recovery_io_objects 1522`),
				regexp.MustCompile(`client_io_ops 8710`),
				regexp.MustCompile(`client_io_read_ops 2863`),
				regexp.MustCompile(`client_io_write_ops 5847`),
			},
		},
	} {
		func() {
			collector := NewClusterHealthCollector(NewNoopConn(tt.input))
			if err := prometheus.Register(collector); err != nil {
				t.Fatalf("collector failed to register: %s", err)
			}
			defer prometheus.Unregister(collector)

			server := httptest.NewServer(prometheus.Handler())
			defer server.Close()

			resp, err := http.Get(server.URL)
			if err != nil {
				t.Fatalf("unexpected failed response from prometheus: %s", err)
			}
			defer resp.Body.Close()

			buf, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Fatalf("failed reading server response: %s", err)
			}

			for _, re := range tt.regexes {
				if !re.Match(buf) {
					t.Errorf("failed matching: %q", re)
				}
			}
		}()
	}
}

//.........这里部分代码省略.........
		for !tcpMsgRingStats.Shutdown {
			time.Sleep(time.Minute)
			tcpMsgRingStats = t.Stats(false)
			log.Printf("%v\n", tcpMsgRingStats)
			stats, err := s.gs.Stats(context.Background(), false)
			if err != nil {
				log.Printf("stats error: %s\n", err)
			} else if s, ok := stats.(*store.GroupStoreStats); ok {
				mValues.Set(float64(s.Values))
				mValueBytes.Set(float64(s.ValueBytes))
				mLookups.Add(float64(s.Lookups))
				mLookupErrors.Add(float64(s.LookupErrors))
				mLookupGroups.Add(float64(s.LookupGroups))
				mLookupGroupItems.Add(float64(s.LookupGroupItems))
				mLookupGroupErrors.Add(float64(s.LookupGroupErrors))
				mReads.Add(float64(s.Reads))
				mReadErrors.Add(float64(s.ReadErrors))
				mReadGroups.Add(float64(s.ReadGroups))
				mReadGroupItems.Add(float64(s.ReadGroupItems))
				mReadGroupErrors.Add(float64(s.ReadGroupErrors))
				mWrites.Add(float64(s.Writes))
				mWriteErrors.Add(float64(s.WriteErrors))
				mWritesOverridden.Add(float64(s.WritesOverridden))
				mDeletes.Add(float64(s.Deletes))
				mDeleteErrors.Add(float64(s.DeleteErrors))
				mDeletesOverridden.Add(float64(s.DeletesOverridden))
				mOutBulkSets.Add(float64(s.OutBulkSets))
				mOutBulkSetValues.Add(float64(s.OutBulkSetValues))
				mOutBulkSetPushes.Add(float64(s.OutBulkSetPushes))
				mOutBulkSetPushValues.Add(float64(s.OutBulkSetPushValues))
				mInBulkSets.Add(float64(s.InBulkSets))
				mInBulkSetDrops.Add(float64(s.InBulkSetDrops))
				mInBulkSetInvalids.Add(float64(s.InBulkSetInvalids))
				mInBulkSetWrites.Add(float64(s.InBulkSetWrites))
				mInBulkSetWriteErrors.Add(float64(s.InBulkSetWriteErrors))
				mInBulkSetWritesOverridden.Add(float64(s.InBulkSetWritesOverridden))
				mOutBulkSetAcks.Add(float64(s.OutBulkSetAcks))
				mInBulkSetAcks.Add(float64(s.InBulkSetAcks))
				mInBulkSetAckDrops.Add(float64(s.InBulkSetAckDrops))
				mInBulkSetAckInvalids.Add(float64(s.InBulkSetAckInvalids))
				mInBulkSetAckWrites.Add(float64(s.InBulkSetAckWrites))
				mInBulkSetAckWriteErrors.Add(float64(s.InBulkSetAckWriteErrors))
				mInBulkSetAckWr